From c5fdbdae321903740e0e735aa89fab5647992687 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 19 Jun 2023 16:54:05 +0200
Subject: [PATCH 001/421] LocalStore::addTempRoot(): Handle ENOENT

If the garbage collector has acquired the global GC lock, but hasn't
created the GC socket yet, then a client attempting to connect would
get ENOENT. Note that this only happens when the GC runs for the first
time on a machine. Subsequently clients will get ECONNREFUSED which
was already handled.

Fixes #7370.
---
 src/libstore/gc.cc       | 13 +++++++++----
 tests/gc-non-blocking.sh |  7 ++++++-
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 0038ec802..b5b9e2049 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -138,9 +138,9 @@ void LocalStore::addTempRoot(const StorePath & path)
             try {
                 nix::connect(fdRootsSocket->get(), socketPath);
             } catch (SysError & e) {
-                /* The garbage collector may have exited, so we need to
-                   restart. */
-                if (e.errNo == ECONNREFUSED) {
+                /* The garbage collector may have exited or not
+                   created the socket yet, so we need to restart. */
+                if (e.errNo == ECONNREFUSED || e.errNo == ENOENT) {
                     debug("GC socket connection refused");
                     fdRootsSocket->close();
                     goto restart;
@@ -503,6 +503,11 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     auto fdGCLock = openGCLock();
     FdLock gcLock(fdGCLock.get(), ltWrite, true, "waiting for the big garbage collector lock...");
 
+    /* Synchronisation point to test ENOENT handling in
+       addTempRoot(), see tests/gc-non-blocking.sh. */
+    if (auto p = getEnv("_NIX_TEST_GC_SYNC_2"))
+        readFile(*p);
+
     /* Start the server for receiving new roots. */
     auto socketPath = stateDir.get() + gcSocketPath;
     createDirs(dirOf(socketPath));
@@ -772,7 +777,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
         }
     };
 
-    /* Synchronisation point for testing, see tests/gc-concurrent.sh. */
+    /* Synchronisation point for testing, see tests/gc-non-blocking.sh. */
     if (auto p = getEnv("_NIX_TEST_GC_SYNC"))
         readFile(*p);
 
diff --git a/tests/gc-non-blocking.sh b/tests/gc-non-blocking.sh
index 0d781485d..da6dbdf5d 100644
--- a/tests/gc-non-blocking.sh
+++ b/tests/gc-non-blocking.sh
@@ -9,16 +9,21 @@ clearStore
 fifo=$TEST_ROOT/test.fifo
 mkfifo "$fifo"
 
+fifo2=$TEST_ROOT/test2.fifo
+mkfifo "$fifo2"
+
 dummy=$(nix store add-path ./simple.nix)
 
 running=$TEST_ROOT/running
 touch $running
 
-(_NIX_TEST_GC_SYNC=$fifo nix-store --gc -vvvvv; rm $running) &
+(_NIX_TEST_GC_SYNC=$fifo _NIX_TEST_GC_SYNC_2=$fifo2 nix-store --gc -vvvvv; rm $running) &
 pid=$!
 
 sleep 2
 
+(sleep 1; echo > $fifo2) &
+
 outPath=$(nix-build --max-silent-time 60 -o "$TEST_ROOT/result" -E "
   with import ./config.nix;
   mkDerivation {

From 3859b425975d0347e724b6abb513662667b3e8c7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 21 Jun 2023 16:17:21 +0200
Subject: [PATCH 002/421] Wait for pid

---
 tests/gc-non-blocking.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/gc-non-blocking.sh b/tests/gc-non-blocking.sh
index da6dbdf5d..7f2aebb8b 100644
--- a/tests/gc-non-blocking.sh
+++ b/tests/gc-non-blocking.sh
@@ -23,6 +23,7 @@ pid=$!
 sleep 2
 
 (sleep 1; echo > $fifo2) &
+pid2=$!
 
 outPath=$(nix-build --max-silent-time 60 -o "$TEST_ROOT/result" -E "
   with import ./config.nix;
@@ -32,6 +33,7 @@ outPath=$(nix-build --max-silent-time 60 -o "$TEST_ROOT/result" -E "
   }")
 
 wait $pid
+wait $pid2
 
 (! test -e $running)
 (! test -e $dummy)

From a2f0ba6a6dbb79efbb83ebe92b287f79b3f3af91 Mon Sep 17 00:00:00 2001
From: Vladimir Kryachko <vkryachko@google.com>
Date: Mon, 16 Oct 2023 16:51:49 -0400
Subject: [PATCH 003/421] Fix transitive input locking.

Fixes reproducibility issue described in #9143

Fixes #9143
---
 src/libexpr/flake/flake.cc | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index a6212c12f..5d9d60655 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -351,10 +351,13 @@ LockedFlake lockFlake(
         debug("old lock file: %s", oldLockFile);
 
         std::map<InputPath, FlakeInput> overrides;
+        std::set<InputPath> explicitCliOverrides;
         std::set<InputPath> overridesUsed, updatesUsed;
 
-        for (auto & i : lockFlags.inputOverrides)
+        for (auto & i : lockFlags.inputOverrides) {
             overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second });
+            explicitCliOverrides.insert(i.first);
+        }
 
         LockFile newLockFile;
 
@@ -425,6 +428,7 @@ LockedFlake lockFlake(
                        ancestors? */
                     auto i = overrides.find(inputPath);
                     bool hasOverride = i != overrides.end();
+                    bool hasCliOverride = explicitCliOverrides.find(inputPath) != explicitCliOverrides.end();
                     if (hasOverride) {
                         overridesUsed.insert(inputPath);
                         // Respect the “flakeness” of the input even if we
@@ -460,7 +464,7 @@ LockedFlake lockFlake(
 
                     if (oldLock
                         && oldLock->originalRef == *input.ref
-                        && !hasOverride)
+                        && !hasCliOverride)
                     {
                         debug("keeping existing input '%s'", inputPathS);
 
@@ -541,7 +545,7 @@ LockedFlake lockFlake(
                             nuked the next time we update the lock
                             file. That is, overrides are sticky unless you
                             use --no-write-lock-file. */
-                        auto ref = input2.ref ? *input2.ref : *input.ref;
+                        auto ref = (input2.ref && explicitCliOverrides.contains(inputPath)) ? *input2.ref : *input.ref;
 
                         if (input.isFlake) {
                             Path localPath = parentPath;

From 311e2ad024441950cb1300e56c9745259deebdda Mon Sep 17 00:00:00 2001
From: Vladimir Kryachko <vkryachko@google.com>
Date: Wed, 18 Oct 2023 10:37:06 -0400
Subject: [PATCH 004/421] Address review comments

---
 src/libexpr/flake/flake.cc              |  2 +-
 tests/functional/flakes/follow-paths.sh | 76 +++++++++++++++++++++++++
 2 files changed, 77 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 5d9d60655..2c7e12ec9 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -428,7 +428,7 @@ LockedFlake lockFlake(
                        ancestors? */
                     auto i = overrides.find(inputPath);
                     bool hasOverride = i != overrides.end();
-                    bool hasCliOverride = explicitCliOverrides.find(inputPath) != explicitCliOverrides.end();
+                    bool hasCliOverride = explicitCliOverrides.contains(inputPath);
                     if (hasOverride) {
                         overridesUsed.insert(inputPath);
                         // Respect the “flakeness” of the input even if we
diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh
index 8573b5511..7f4e8bf5d 100644
--- a/tests/functional/flakes/follow-paths.sh
+++ b/tests/functional/flakes/follow-paths.sh
@@ -260,3 +260,79 @@ EOF
 
 checkRes=$(nix flake lock "$flakeFollowCycle" 2>&1 && fail "nix flake lock should have failed." || true)
 echo $checkRes | grep -F "error: follow cycle detected: [baz -> foo -> bar -> baz]"
+
+
+# Test transitive input url locking
+# This tests the following lockfile issue: https://github.com/NixOS/nix/issues/9143
+#
+# We construct the following graph, where p->q means p has input q.
+#
+# A -> B -> C
+#
+# And override B/C to flake D, first in A's flake.nix and then with --override-input.
+#
+# A -> B -> D
+flakeFollowsCustomUrlA="$TEST_ROOT/follows/custom-url/flakeA"
+flakeFollowsCustomUrlB="$TEST_ROOT/follows/custom-url/flakeA/flakeB"
+flakeFollowsCustomUrlC="$TEST_ROOT/follows/custom-url/flakeA/flakeB/flakeC"
+flakeFollowsCustomUrlD="$TEST_ROOT/follows/custom-url/flakeA/flakeB/flakeD"
+
+
+createGitRepo "$flakeFollowsCustomUrlA"
+mkdir -p "$flakeFollowsCustomUrlB"
+mkdir -p "$flakeFollowsCustomUrlC"
+mkdir -p "$flakeFollowsCustomUrlD"
+
+cat > "$flakeFollowsCustomUrlD/flake.nix" <<EOF
+{
+    description = "Flake D";
+    inputs = {};
+    outputs = { ... }: {};
+}
+EOF
+
+cat > "$flakeFollowsCustomUrlC/flake.nix" <<EOF
+{
+    description = "Flake C";
+    inputs = {};
+    outputs = { ... }: {};
+}
+EOF
+
+cat > "$flakeFollowsCustomUrlB/flake.nix" <<EOF
+{
+    description = "Flake B";
+    inputs = {
+        C = {
+            url = "path:./flakeC";
+        };
+    };
+    outputs = { ... }: {};
+}
+EOF
+
+cat > "$flakeFollowsCustomUrlA/flake.nix" <<EOF
+{
+    description = "Flake A";
+    inputs = {
+        B = {
+            url = "path:./flakeB";
+            inputs.C.url = "path:./flakeB/flakeD";
+        };
+    };
+    outputs = { ... }: {};
+}
+EOF
+
+git -C "$flakeFollowsCustomUrlA" add flake.nix flakeB/flake.nix \
+  flakeB/flakeC/flake.nix flakeB/flakeD/flake.nix
+
+# lock "original" entry should contain overridden url
+json=$(nix flake metadata "$flakeFollowsCustomUrlA" --json)
+[[ $(echo "$json" | jq -r .locks.nodes.C.original.path) = './flakeB/flakeD' ]]
+rm "$flakeFollowsCustomUrlA"/flake.lock
+
+# if override-input is specified, lock "original" entry should contain original url
+json=$(nix flake metadata "$flakeFollowsCustomUrlA" --override-input B/C "path:./flakeB/flakeD" --json)
+echo "$json" | jq .locks.nodes.C.original
+[[ $(echo "$json" | jq -r .locks.nodes.C.original.path) = './flakeC' ]]

From 1d28d613b1d0447b60de2d2044aeac3e1d543aa6 Mon Sep 17 00:00:00 2001
From: Bouke van der Bijl <i@bou.ke>
Date: Wed, 25 Oct 2023 11:39:18 +0200
Subject: [PATCH 005/421] config: add included files into parsedContents before
 applying

Fixes #8719
---
 src/libutil/config.cc    | 16 ++++++++++++----
 tests/functional/init.sh |  3 ++-
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 8e06273ee..17380b6d8 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -88,10 +88,9 @@ void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overridd
             res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description});
 }
 
-void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) {
-    unsigned int pos = 0;
 
-    std::vector<std::pair<std::string, std::string>> parsedContents;
+static void applyConfigInner(const std::string & contents, const std::string & path, std::vector<std::pair<std::string, std::string>> & parsedContents) {
+    unsigned int pos = 0;
 
     while (pos < contents.size()) {
         std::string line;
@@ -123,7 +122,10 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string
                 throw UsageError("illegal configuration line '%1%' in '%2%'", line, path);
             auto p = absPath(tokens[1], dirOf(path));
             if (pathExists(p)) {
-                applyConfigFile(p);
+                try {
+                    std::string includedContents = readFile(path);
+                    applyConfigInner(includedContents, p, parsedContents);
+                } catch (SysError &) { }
             } else if (!ignoreMissing) {
                 throw Error("file '%1%' included from '%2%' not found", p, path);
             }
@@ -143,6 +145,12 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string
             concatStringsSep(" ", Strings(i, tokens.end())),
         });
     };
+}
+
+void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) {
+    std::vector<std::pair<std::string, std::string>> parsedContents;
+
+    applyConfigInner(contents, path, parsedContents);
 
     // First apply experimental-feature related settings
     for (auto & [name, value] : parsedContents)
diff --git a/tests/functional/init.sh b/tests/functional/init.sh
index c420e8c9f..d697b1a30 100755
--- a/tests/functional/init.sh
+++ b/tests/functional/init.sh
@@ -20,7 +20,7 @@ cat > "$NIX_CONF_DIR"/nix.conf <<EOF
 build-users-group =
 keep-derivations = false
 sandbox = false
-experimental-features = nix-command flakes
+experimental-features = nix-command
 gc-reserved-space = 0
 substituters =
 flake-registry = $TEST_ROOT/registry.json
@@ -31,6 +31,7 @@ EOF
 
 cat > "$NIX_CONF_DIR"/nix.conf.extra <<EOF
 fsync-metadata = false
+extra-experimental-features = flakes
 !include nix.conf.extra.not-there
 EOF
 

From e1b8442fa1dbd2e69598dbeb701da4df8e6d2c38 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 24 Oct 2023 08:20:31 +0200
Subject: [PATCH 006/421] Fetcher cache: Add support for caching facts not
 related to store paths

---
 src/libfetchers/cache.cc | 57 ++++++++++++++++++++++++++++++++++++++++
 src/libfetchers/cache.hh | 38 +++++++++++++++++++++++++++
 2 files changed, 95 insertions(+)

diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc
index 0c8ecac9d..8a3e462d3 100644
--- a/src/libfetchers/cache.cc
+++ b/src/libfetchers/cache.cc
@@ -19,6 +19,9 @@ create table if not exists Cache (
 );
 )sql";
 
+// FIXME: we should periodically purge/nuke this cache to prevent it
+// from growing too big.
+
 struct CacheImpl : Cache
 {
     struct State
@@ -47,6 +50,60 @@ struct CacheImpl : Cache
             "select info, path, immutable, timestamp from Cache where input = ?");
     }
 
+    void upsert(
+        const Attrs & inAttrs,
+        const Attrs & infoAttrs) override
+    {
+        _state.lock()->add.use()
+            (attrsToJSON(inAttrs).dump())
+            (attrsToJSON(infoAttrs).dump())
+            ("") // no path
+            (false)
+            (time(0)).exec();
+    }
+
+    std::optional<Attrs> lookup(const Attrs & inAttrs) override
+    {
+        if (auto res = lookupExpired(inAttrs))
+            return std::move(res->infoAttrs);
+        return {};
+    }
+
+    std::optional<Attrs> lookupWithTTL(const Attrs & inAttrs) override
+    {
+        if (auto res = lookupExpired(inAttrs)) {
+            if (!res->expired)
+                return std::move(res->infoAttrs);
+            debug("ignoring expired cache entry '%s'",
+                attrsToJSON(inAttrs).dump());
+        }
+        return {};
+    }
+
+    std::optional<Result2> lookupExpired(const Attrs & inAttrs) override
+    {
+        auto state(_state.lock());
+
+        auto inAttrsJSON = attrsToJSON(inAttrs).dump();
+
+        auto stmt(state->lookup.use()(inAttrsJSON));
+        if (!stmt.next()) {
+            debug("did not find cache entry for '%s'", inAttrsJSON);
+            return {};
+        }
+
+        auto infoJSON = stmt.getStr(0);
+        auto locked = stmt.getInt(2) != 0;
+        auto timestamp = stmt.getInt(3);
+
+        debug("using cache entry '%s' -> '%s'", inAttrsJSON, infoJSON);
+
+        return Result2 {
+            .expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),
+            .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJSON)),
+        };
+    }
+
     void add(
         ref<Store> store,
         const Attrs & inAttrs,
diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh
index af34e66ce..b517d496e 100644
--- a/src/libfetchers/cache.hh
+++ b/src/libfetchers/cache.hh
@@ -10,6 +10,44 @@ struct Cache
 {
     virtual ~Cache() { }
 
+    /* A cache for arbitrary Attrs -> Attrs mappings with a timestamp
+       for expiration. */
+
+    /*
+     * Add a value to the cache. The cache is an arbitrary mapping of
+     * Attrs to Attrs.
+     */
+    virtual void upsert(
+        const Attrs & inAttrs,
+        const Attrs & infoAttrs) = 0;
+
+    /*
+     * Look up a key with infinite TTL.
+     */
+    virtual std::optional<Attrs> lookup(
+        const Attrs & inAttrs) = 0;
+
+    /*
+     * Look up a key. Return nothing if its TTL has exceeded
+     * `settings.tarballTTL`.
+     */
+    virtual std::optional<Attrs> lookupWithTTL(
+        const Attrs & inAttrs) = 0;
+
+    struct Result2
+    {
+        bool expired = false;
+        Attrs infoAttrs;
+    };
+
+    /*
+     * Look up a key. Return a bool denoting whether its TTL has
+     * exceeded `settings.tarballTTL`.
+     */
+    virtual std::optional<Result2> lookupExpired(
+        const Attrs & inAttrs) = 0;
+
+    /* Old cache for things that have a store path. */
     virtual void add(
         ref<Store> store,
         const Attrs & inAttrs,

From 1d0e3d84b6ed693c140c3b7fd6a72ef8a8a26ec3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 24 Oct 2023 10:23:46 +0200
Subject: [PATCH 007/421] Provide a InputScheme::fetch() built on top of
 InputScheme::getAccessor()

This is for graceful migration to lazy-trees fetchers (which are all
accessor-based). Eventually fetch() will be removed.
---
 src/libfetchers/fetchers.cc | 13 +++++++++++++
 src/libfetchers/fetchers.hh |  6 ++++--
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index c339c441b..3e654dd53 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -1,5 +1,6 @@
 #include "fetchers.hh"
 #include "store-api.hh"
+#include "input-accessor.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -312,6 +313,18 @@ void InputScheme::clone(const Input & input, const Path & destDir) const
     throw Error("do not know how to clone input '%s'", input.to_string());
 }
 
+std::pair<StorePath, Input> InputScheme::fetch(ref<Store> store, const Input & input)
+{
+    auto [accessor, input2] = getAccessor(store, input);
+    auto storePath = accessor->root().fetchToStore(store, input2.getName());
+    return {storePath, input2};
+}
+
+std::pair<ref<InputAccessor>, Input> InputScheme::getAccessor(ref<Store> store, const Input & input) const
+{
+    throw UnimplementedError("InputScheme must implement fetch() or getAccessor()");
+}
+
 std::optional<ExperimentalFeature> InputScheme::experimentalFeature()
 {
     return {};
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index 4212a3e1f..7b70ab6e2 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -9,7 +9,7 @@
 
 #include <memory>
 
-namespace nix { class Store; class StorePath; }
+namespace nix { class Store; class StorePath; struct InputAccessor; }
 
 namespace nix::fetchers {
 
@@ -148,7 +148,9 @@ struct InputScheme
         std::string_view contents,
         std::optional<std::string> commitMsg) const;
 
-    virtual std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) = 0;
+    virtual std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input);
+
+    virtual std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & input) const;
 
     /**
      * Is this `InputScheme` part of an experimental feature?

From ee36a44bf272c8cca62a2ce96a017a8150c4d35b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 25 Oct 2023 18:55:08 +0200
Subject: [PATCH 008/421] GitInputScheme: Use libgit2

This replaces most calls to the "git" binary with libgit2.
---
 flake.nix                                     |   1 +
 src/libfetchers/git-utils.cc                  | 498 ++++++++++++++
 src/libfetchers/git-utils.hh                  |  56 ++
 src/libfetchers/git.cc                        | 609 ++++++++----------
 src/libfetchers/local.mk                      |   2 +-
 tests/functional/fetchGit.sh                  |   7 +-
 tests/functional/flakes/flake-in-submodule.sh |   5 +-
 7 files changed, 839 insertions(+), 339 deletions(-)
 create mode 100644 src/libfetchers/git-utils.cc
 create mode 100644 src/libfetchers/git-utils.hh

diff --git a/flake.nix b/flake.nix
index 398ba10a0..3472bf7a8 100644
--- a/flake.nix
+++ b/flake.nix
@@ -191,6 +191,7 @@
             bzip2 xz brotli editline
             openssl sqlite
             libarchive
+            libgit2
             boost
             lowdown-nix
             libsodium
diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
new file mode 100644
index 000000000..68e39580f
--- /dev/null
+++ b/src/libfetchers/git-utils.cc
@@ -0,0 +1,498 @@
+#include "git-utils.hh"
+#include "input-accessor.hh"
+#include "cache.hh"
+
+#include <boost/core/span.hpp>
+
+#include <git2/blob.h>
+#include <git2/commit.h>
+#include <git2/describe.h>
+#include <git2/errors.h>
+#include <git2/global.h>
+#include <git2/object.h>
+#include <git2/refs.h>
+#include <git2/remote.h>
+#include <git2/repository.h>
+#include <git2/status.h>
+#include <git2/tree.h>
+
+#include <unordered_set>
+#include <queue>
+
+namespace std {
+
+template<> struct hash<git_oid>
+{
+    size_t operator()(const git_oid & oid) const
+    {
+        return * (size_t *) oid.id;
+    }
+};
+
+}
+
+std::ostream & operator << (std::ostream & str, const git_oid & oid)
+{
+    str << git_oid_tostr_s(&oid);
+    return str;
+}
+
+bool operator == (const git_oid & oid1, const git_oid & oid2)
+{
+    return git_oid_equal(&oid1, &oid2);
+}
+
+namespace nix {
+
+// Some wrapper types that ensure that the git_*_free functions get called.
+template<auto del>
+struct Deleter
+{
+    template <typename T>
+    void operator()(T * p) const { del(p); };
+};
+
+typedef std::unique_ptr<git_repository, Deleter<git_repository_free>> Repository;
+typedef std::unique_ptr<git_tree_entry, Deleter<git_tree_entry_free>> TreeEntry;
+typedef std::unique_ptr<git_tree, Deleter<git_tree_free>> Tree;
+typedef std::unique_ptr<git_treebuilder, Deleter<git_treebuilder_free>> TreeBuilder;
+typedef std::unique_ptr<git_blob, Deleter<git_blob_free>> Blob;
+typedef std::unique_ptr<git_object, Deleter<git_object_free>> Object;
+typedef std::unique_ptr<git_commit, Deleter<git_commit_free>> Commit;
+typedef std::unique_ptr<git_reference, Deleter<git_reference_free>> Reference;
+typedef std::unique_ptr<git_describe_result, Deleter<git_describe_result_free>> DescribeResult;
+typedef std::unique_ptr<git_status_list, Deleter<git_status_list_free>> StatusList;
+typedef std::unique_ptr<git_remote, Deleter<git_remote_free>> Remote;
+
+// A helper to ensure that we don't leak objects returned by libgit2.
+template<typename T>
+struct Setter
+{
+    T & t;
+    typename T::pointer p = nullptr;
+
+    Setter(T & t) : t(t) { }
+
+    ~Setter() { if (p) t = T(p); }
+
+    operator typename T::pointer * () { return &p; }
+};
+
+Hash toHash(const git_oid & oid)
+{
+    #ifdef GIT_EXPERIMENTAL_SHA256
+    assert(oid.type == GIT_OID_SHA1);
+    #endif
+    Hash hash(htSHA1);
+    memcpy(hash.hash, oid.id, hash.hashSize);
+    return hash;
+}
+
+static void initLibGit2()
+{
+    if (git_libgit2_init() < 0)
+        throw Error("initialising libgit2: %s", git_error_last()->message);
+}
+
+git_oid hashToOID(const Hash & hash)
+{
+    git_oid oid;
+    if (git_oid_fromstr(&oid, hash.gitRev().c_str()))
+        throw Error("cannot convert '%s' to a Git OID", hash.gitRev());
+    return oid;
+}
+
+Object lookupObject(git_repository * repo, const git_oid & oid)
+{
+    Object obj;
+    if (git_object_lookup(Setter(obj), repo, &oid, GIT_OBJECT_ANY)) {
+        auto err = git_error_last();
+        throw Error("getting Git object '%s': %s", oid, err->message);
+    }
+    return obj;
+}
+
+template<typename T>
+T peelObject(git_repository * repo, git_object * obj, git_object_t type)
+{
+    T obj2;
+    if (git_object_peel((git_object * *) (typename T::pointer *) Setter(obj2), obj, type)) {
+        auto err = git_error_last();
+        throw Error("peeling Git object '%s': %s", git_object_id(obj), err->message);
+    }
+    return obj2;
+}
+
+int statusCallbackTrampoline(const char * path, unsigned int statusFlags, void * payload)
+{
+    return (*((std::function<int(const char * path, unsigned int statusFlags)> *) payload))(path, statusFlags);
+}
+
+struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
+{
+    CanonPath path;
+    Repository repo;
+
+    GitRepoImpl(CanonPath _path, bool create, bool bare)
+        : path(std::move(_path))
+    {
+        initLibGit2();
+
+        if (pathExists(path.abs())) {
+            if (git_repository_open(Setter(repo), path.c_str()))
+                throw Error("opening Git repository '%s': %s", path, git_error_last()->message);
+        } else {
+            if (git_repository_init(Setter(repo), path.c_str(), bare))
+                throw Error("creating Git repository '%s': %s", path, git_error_last()->message);
+        }
+    }
+
+    operator git_repository * ()
+    {
+        return repo.get();
+    }
+
+    uint64_t getRevCount(const Hash & rev) override
+    {
+        std::unordered_set<git_oid> done;
+        std::queue<Commit> todo;
+
+        todo.push(peelObject<Commit>(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT));
+
+        while (auto commit = pop(todo)) {
+            if (!done.insert(*git_commit_id(commit->get())).second) continue;
+
+            for (size_t n = 0; n < git_commit_parentcount(commit->get()); ++n) {
+                git_commit * parent;
+                if (git_commit_parent(&parent, commit->get(), n))
+                    throw Error("getting parent of Git commit '%s': %s", *git_commit_id(commit->get()), git_error_last()->message);
+                todo.push(Commit(parent));
+            }
+        }
+
+        return done.size();
+    }
+
+    uint64_t getLastModified(const Hash & rev) override
+    {
+        auto commit = peelObject<Commit>(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT);
+
+        return git_commit_time(commit.get());
+    }
+
+    bool isShallow() override
+    {
+        return git_repository_is_shallow(*this);
+    }
+
+    Hash resolveRef(std::string ref) override
+    {
+        // Handle revisions used as refs.
+        {
+            git_oid oid;
+            if (git_oid_fromstr(&oid, ref.c_str()) == 0)
+                return toHash(oid);
+        }
+
+        // Resolve short names like 'master'.
+        Reference ref2;
+        if (!git_reference_dwim(Setter(ref2), *this, ref.c_str()))
+            ref = git_reference_name(ref2.get());
+
+        // Resolve full references like 'refs/heads/master'.
+        Reference ref3;
+        if (git_reference_lookup(Setter(ref3), *this, ref.c_str()))
+            throw Error("resolving Git reference '%s': %s", ref, git_error_last()->message);
+
+        auto oid = git_reference_target(ref3.get());
+        if (!oid)
+            throw Error("cannot get OID for Git reference '%s'", git_reference_name(ref3.get()));
+
+        return toHash(*oid);
+    }
+
+    WorkdirInfo getWorkdirInfo() override
+    {
+        WorkdirInfo info;
+
+        /* Get the head revision, if any. */
+        git_oid headRev;
+        if (auto err = git_reference_name_to_id(&headRev, *this, "HEAD")) {
+            if (err != GIT_ENOTFOUND)
+                throw Error("resolving HEAD: %s", git_error_last()->message);
+        } else
+            info.headRev = toHash(headRev);
+
+        /* Get all tracked files and determine whether the working
+           directory is dirty. */
+        std::function<int(const char * path, unsigned int statusFlags)> statusCallback = [&](const char * path, unsigned int statusFlags)
+        {
+            if (!(statusFlags & GIT_STATUS_INDEX_DELETED) &&
+                !(statusFlags & GIT_STATUS_WT_DELETED))
+                info.files.insert(CanonPath(path));
+            if (statusFlags != GIT_STATUS_CURRENT)
+                info.isDirty = true;
+            return 0;
+        };
+
+        git_status_options options = GIT_STATUS_OPTIONS_INIT;
+        options.flags |= GIT_STATUS_OPT_INCLUDE_UNMODIFIED;
+        options.flags |= GIT_STATUS_OPT_EXCLUDE_SUBMODULES;
+        if (git_status_foreach_ext(*this, &options, &statusCallbackTrampoline, &statusCallback))
+            throw Error("getting working directory status: %s", git_error_last()->message);
+
+        return info;
+    }
+
+    std::optional<std::string> getWorkdirRef() override
+    {
+        Reference ref;
+        if (git_reference_lookup(Setter(ref), *this, "HEAD"))
+            throw Error("looking up HEAD: %s", git_error_last()->message);
+
+        if (auto target = git_reference_symbolic_target(ref.get()))
+            return target;
+
+        return std::nullopt;
+    }
+
+    bool hasObject(const Hash & oid_) override
+    {
+        auto oid = hashToOID(oid_);
+
+        Object obj;
+        if (auto errCode = git_object_lookup(Setter(obj), *this, &oid, GIT_OBJECT_ANY)) {
+            if (errCode == GIT_ENOTFOUND) return false;
+            auto err = git_error_last();
+            throw Error("getting Git object '%s': %s", oid, err->message);
+        }
+
+        return true;
+    }
+
+    ref<InputAccessor> getAccessor(const Hash & rev) override;
+
+    void fetch(
+        const std::string & url,
+        const std::string & refspec) override
+    {
+        /* FIXME: use libgit2. Unfortunately, it doesn't support
+           ssh_config at the moment. */
+        #if 0
+        Remote remote;
+
+        if (git_remote_create_anonymous(Setter(remote), *this, url.c_str()))
+            throw Error("cannot create Git remote '%s': %s", url, git_error_last()->message);
+
+        char * refspecs[] = {(char *) refspec.c_str()};
+        git_strarray refspecs2 {
+            .strings = refspecs,
+            .count = 1
+        };
+
+        if (git_remote_fetch(remote.get(), &refspecs2, nullptr, nullptr))
+            throw Error("fetching '%s' from '%s': %s", refspec, url, git_error_last()->message);
+        #endif
+
+        // FIXME: git stderr messes up our progress indicator, so
+        // we're using --quiet for now. Should process its stderr.
+        runProgram("git", true,
+            { "-C", path.abs(),
+              "--bare",
+              "fetch",
+              "--quiet",
+              "--force",
+              "--",
+              url,
+              refspec
+            }, {}, true);
+    }
+};
+
+ref<GitRepo> GitRepo::openRepo(const CanonPath & path, bool create, bool bare)
+{
+    return make_ref<GitRepoImpl>(path, create, bare);
+}
+
+struct GitInputAccessor : InputAccessor
+{
+    ref<GitRepoImpl> repo;
+    Tree root;
+
+    GitInputAccessor(ref<GitRepoImpl> repo_, const Hash & rev)
+        : repo(repo_)
+        , root(peelObject<Tree>(*repo, lookupObject(*repo, hashToOID(rev)).get(), GIT_OBJECT_TREE))
+    {
+    }
+
+    std::string readBlob(const CanonPath & path, bool symlink)
+    {
+        auto blob = getBlob(path, symlink);
+
+        auto data = std::string_view((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get()));
+
+        return std::string(data);
+    }
+
+    std::string readFile(const CanonPath & path) override
+    {
+        return readBlob(path, false);
+    }
+
+    bool pathExists(const CanonPath & path) override
+    {
+        return path.isRoot() ? true : (bool) lookup(path);
+    }
+
+    Stat lstat(const CanonPath & path) override
+    {
+        if (path.isRoot())
+            return Stat { .type = tDirectory };
+
+        auto entry = need(path);
+
+        auto mode = git_tree_entry_filemode(entry);
+
+        if (mode == GIT_FILEMODE_TREE)
+            return Stat { .type = tDirectory };
+
+        else if (mode == GIT_FILEMODE_BLOB)
+            return Stat { .type = tRegular };
+
+        else if (mode == GIT_FILEMODE_BLOB_EXECUTABLE)
+            return Stat { .type = tRegular, .isExecutable = true };
+
+        else if (mode == GIT_FILEMODE_LINK)
+            return Stat { .type = tSymlink };
+
+        else if (mode == GIT_FILEMODE_COMMIT)
+            // Treat submodules as an empty directory.
+            return Stat { .type = tDirectory };
+
+        else
+            throw Error("file '%s' has an unsupported Git file type");
+    }
+
+    DirEntries readDirectory(const CanonPath & path) override
+    {
+        return std::visit(overloaded {
+            [&](Tree tree) {
+                DirEntries res;
+
+                auto count = git_tree_entrycount(tree.get());
+
+                for (size_t n = 0; n < count; ++n) {
+                    auto entry = git_tree_entry_byindex(tree.get(), n);
+                    // FIXME: add to cache
+                    res.emplace(std::string(git_tree_entry_name(entry)), DirEntry{});
+                }
+
+                return res;
+            },
+            [&](Submodule) {
+                return DirEntries();
+            }
+        }, getTree(path));
+    }
+
+    std::string readLink(const CanonPath & path) override
+    {
+        return readBlob(path, true);
+    }
+
+    std::map<CanonPath, TreeEntry> lookupCache;
+
+    /* Recursively look up 'path' relative to the root. */
+    git_tree_entry * lookup(const CanonPath & path)
+    {
+        if (path.isRoot()) return nullptr;
+
+        auto i = lookupCache.find(path);
+        if (i == lookupCache.end()) {
+            TreeEntry entry;
+            if (auto err = git_tree_entry_bypath(Setter(entry), root.get(), std::string(path.rel()).c_str())) {
+                if (err != GIT_ENOTFOUND)
+                    throw Error("looking up '%s': %s", showPath(path), git_error_last()->message);
+            }
+
+            i = lookupCache.emplace(path, std::move(entry)).first;
+        }
+
+        return &*i->second;
+    }
+
+    git_tree_entry * need(const CanonPath & path)
+    {
+        auto entry = lookup(path);
+        if (!entry)
+            throw Error("'%s' does not exist", showPath(path));
+        return entry;
+    }
+
+    struct Submodule { };
+
+    std::variant<Tree, Submodule> getTree(const CanonPath & path)
+    {
+        if (path.isRoot()) {
+            Tree tree;
+            if (git_tree_dup(Setter(tree), root.get()))
+                throw Error("duplicating directory '%s': %s", showPath(path), git_error_last()->message);
+            return tree;
+        }
+
+        auto entry = need(path);
+
+        if (git_tree_entry_type(entry) == GIT_OBJECT_COMMIT)
+            return Submodule();
+
+        if (git_tree_entry_type(entry) != GIT_OBJECT_TREE)
+            throw Error("'%s' is not a directory", showPath(path));
+
+        Tree tree;
+        if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(tree), *repo, entry))
+            throw Error("looking up directory '%s': %s", showPath(path), git_error_last()->message);
+
+        return tree;
+    }
+
+    Blob getBlob(const CanonPath & path, bool expectSymlink)
+    {
+        auto notExpected = [&]()
+        {
+            throw Error(
+                expectSymlink
+                ? "'%s' is not a symlink"
+                : "'%s' is not a regular file",
+                showPath(path));
+        };
+
+        if (path.isRoot()) notExpected();
+
+        auto entry = need(path);
+
+        if (git_tree_entry_type(entry) != GIT_OBJECT_BLOB)
+            notExpected();
+
+        auto mode = git_tree_entry_filemode(entry);
+        if (expectSymlink) {
+            if (mode != GIT_FILEMODE_LINK)
+                notExpected();
+        } else {
+            if (mode != GIT_FILEMODE_BLOB && mode != GIT_FILEMODE_BLOB_EXECUTABLE)
+                notExpected();
+        }
+
+        Blob blob;
+        if (git_tree_entry_to_object((git_object * *) (git_blob * *) Setter(blob), *repo, entry))
+            throw Error("looking up file '%s': %s", showPath(path), git_error_last()->message);
+
+        return blob;
+    }
+};
+
+ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev)
+{
+    return make_ref<GitInputAccessor>(ref<GitRepoImpl>(shared_from_this()), rev);
+}
+
+}
diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh
new file mode 100644
index 000000000..dd2c06672
--- /dev/null
+++ b/src/libfetchers/git-utils.hh
@@ -0,0 +1,56 @@
+#pragma once
+
+#include "input-accessor.hh"
+
+namespace nix {
+
+struct GitRepo
+{
+    virtual ~GitRepo()
+    { }
+
+    static ref<GitRepo> openRepo(const CanonPath & path, bool create = false, bool bare = false);
+
+    virtual uint64_t getRevCount(const Hash & rev) = 0;
+
+    virtual uint64_t getLastModified(const Hash & rev) = 0;
+
+    virtual bool isShallow() = 0;
+
+    /* Return the commit hash to which a ref points. */
+    virtual Hash resolveRef(std::string ref) = 0;
+
+    struct WorkdirInfo
+    {
+        bool isDirty = false;
+
+        /* The checked out commit, or nullopt if there are no commits
+           in the repo yet. */
+        std::optional<Hash> headRev;
+
+        /* All files in the working directory that are unchanged,
+           modified or added, but excluding deleted files. */
+        std::set<CanonPath> files;
+    };
+
+    virtual WorkdirInfo getWorkdirInfo() = 0;
+
+    /* Get the ref that HEAD points to. */
+    virtual std::optional<std::string> getWorkdirRef() = 0;
+
+    struct TarballInfo
+    {
+        Hash treeHash;
+        time_t lastModified;
+    };
+
+    virtual bool hasObject(const Hash & oid) = 0;
+
+    virtual ref<InputAccessor> getAccessor(const Hash & rev) = 0;
+
+    virtual void fetch(
+        const std::string & url,
+        const std::string & refspec) = 0;
+};
+
+}
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 4bfd53b0e..55d3a8ebe 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -7,6 +7,8 @@
 #include "pathlocks.hh"
 #include "util.hh"
 #include "git.hh"
+#include "fs-input-accessor.hh"
+#include "git-utils.hh"
 
 #include "fetch-settings.hh"
 
@@ -137,121 +139,6 @@ bool isNotDotGitDirectory(const Path & path)
     return baseNameOf(path) != ".git";
 }
 
-struct WorkdirInfo
-{
-    bool clean = false;
-    bool hasHead = false;
-};
-
-// Returns whether a git workdir is clean and has commits.
-WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir)
-{
-    const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
-    std::string gitDir(".git");
-
-    auto env = getEnv();
-    // Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong
-    // that way unknown errors can lead to a failure instead of continuing through the wrong code path
-    env["LC_ALL"] = "C";
-
-    /* Check whether HEAD points to something that looks like a commit,
-       since that is the refrence we want to use later on. */
-    auto result = runProgram(RunOptions {
-        .program = "git",
-        .args = { "-C", workdir, "--git-dir", gitDir, "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" },
-        .environment = env,
-        .mergeStderrToStdout = true
-    });
-    auto exitCode = WEXITSTATUS(result.first);
-    auto errorMessage = result.second;
-
-    if (errorMessage.find("fatal: not a git repository") != std::string::npos) {
-        throw Error("'%s' is not a Git repository", workdir);
-    } else if (errorMessage.find("fatal: Needed a single revision") != std::string::npos) {
-        // indicates that the repo does not have any commits
-        // we want to proceed and will consider it dirty later
-    } else if (exitCode != 0) {
-        // any other errors should lead to a failure
-        throw Error("getting the HEAD of the Git tree '%s' failed with exit code %d:\n%s", workdir, exitCode, errorMessage);
-    }
-
-    bool clean = false;
-    bool hasHead = exitCode == 0;
-
-    try {
-        if (hasHead) {
-            // Using git diff is preferrable over lower-level operations here,
-            // because its conceptually simpler and we only need the exit code anyways.
-            auto gitDiffOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "diff", "HEAD", "--quiet"});
-            if (!submodules) {
-                // Changes in submodules should only make the tree dirty
-                // when those submodules will be copied as well.
-                gitDiffOpts.emplace_back("--ignore-submodules");
-            }
-            gitDiffOpts.emplace_back("--");
-            runProgram("git", true, gitDiffOpts);
-
-            clean = true;
-        }
-    } catch (ExecError & e) {
-        if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
-    }
-
-    return WorkdirInfo { .clean = clean, .hasHead = hasHead };
-}
-
-std::pair<StorePath, Input> fetchFromWorkdir(ref<Store> store, Input & input, const Path & workdir, const WorkdirInfo & workdirInfo)
-{
-    const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
-    auto gitDir = ".git";
-
-    if (!fetchSettings.allowDirty)
-        throw Error("Git tree '%s' is dirty", workdir);
-
-    if (fetchSettings.warnDirty)
-        warn("Git tree '%s' is dirty", workdir);
-
-    auto gitOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "ls-files", "-z" });
-    if (submodules)
-        gitOpts.emplace_back("--recurse-submodules");
-
-    auto files = tokenizeString<std::set<std::string>>(
-        runProgram("git", true, gitOpts), "\0"s);
-
-    Path actualPath(absPath(workdir));
-
-    PathFilter filter = [&](const Path & p) -> bool {
-        assert(hasPrefix(p, actualPath));
-        std::string file(p, actualPath.size() + 1);
-
-        auto st = lstat(p);
-
-        if (S_ISDIR(st.st_mode)) {
-            auto prefix = file + "/";
-            auto i = files.lower_bound(prefix);
-            return i != files.end() && hasPrefix(*i, prefix);
-        }
-
-        return files.count(file);
-    };
-
-    auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter);
-
-    // FIXME: maybe we should use the timestamp of the last
-    // modified dirty file?
-    input.attrs.insert_or_assign(
-        "lastModified",
-        workdirInfo.hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
-
-    if (workdirInfo.hasHead) {
-        input.attrs.insert_or_assign("dirtyRev", chomp(
-            runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "HEAD" })) + "-dirty");
-        input.attrs.insert_or_assign("dirtyShortRev", chomp(
-            runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "--short", "HEAD" })) + "-dirty");
-    }
-
-    return {std::move(storePath), input};
-}
 }  // end namespace
 
 struct GitInputScheme : InputScheme
@@ -336,11 +223,11 @@ struct GitInputScheme : InputScheme
 
     void clone(const Input & input, const Path & destDir) const override
     {
-        auto [isLocal, actualUrl] = getActualUrl(input);
+        auto repoInfo = getRepoInfo(input);
 
         Strings args = {"clone"};
 
-        args.push_back(actualUrl);
+        args.push_back(repoInfo.url);
 
         if (auto ref = input.getRef()) {
             args.push_back("--branch");
@@ -356,10 +243,9 @@ struct GitInputScheme : InputScheme
 
     std::optional<Path> getSourcePath(const Input & input) const override
     {
-        auto url = parseURL(getStrAttr(input.attrs, "url"));
-        if (url.scheme == "file" && !input.getRef() && !input.getRev())
-            return url.path;
-        return {};
+        auto repoInfo = getRepoInfo(input);
+        if (repoInfo.isLocal) return repoInfo.url;
+        return std::nullopt;
     }
 
     void putFile(
@@ -368,24 +254,79 @@ struct GitInputScheme : InputScheme
         std::string_view contents,
         std::optional<std::string> commitMsg) const override
     {
-        auto root = getSourcePath(input);
-        if (!root)
+        auto repoInfo = getRepoInfo(input);
+        if (!repoInfo.isLocal)
             throw Error("cannot commit '%s' to Git repository '%s' because it's not a working tree", path, input.to_string());
 
-        writeFile((CanonPath(*root) + path).abs(), contents);
-
-        auto gitDir = ".git";
+        writeFile((CanonPath(repoInfo.url) + path).abs(), contents);
 
         runProgram("git", true,
-            { "-C", *root, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) });
+            { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) });
 
         if (commitMsg)
             runProgram("git", true,
-                { "-C", *root, "--git-dir", gitDir, "commit", std::string(path.rel()), "-m", *commitMsg });
+                { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-m", *commitMsg });
     }
 
-    std::pair<bool, std::string> getActualUrl(const Input & input) const
+    struct RepoInfo
     {
+        bool shallow = false;
+        bool submodules = false;
+        bool allRefs = false;
+
+        std::string cacheType;
+
+        /* Whether this is a local, non-bare repository. */
+        bool isLocal = false;
+
+        /* Working directory info: the complete list of files, and
+           whether the working directory is dirty compared to HEAD. */
+        GitRepo::WorkdirInfo workdirInfo;
+
+        /* URL of the repo, or its path if isLocal. */
+        std::string url;
+
+        void warnDirty() const
+        {
+            if (workdirInfo.isDirty) {
+                if (!fetchSettings.allowDirty)
+                    throw Error("Git tree '%s' is dirty", url);
+
+                if (fetchSettings.warnDirty)
+                    warn("Git tree '%s' is dirty", url);
+            }
+        }
+
+        std::string gitDir = ".git";
+    };
+
+    bool getSubmodulesAttr(const Input & input) const
+    {
+        return maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
+    }
+
+    RepoInfo getRepoInfo(const Input & input) const
+    {
+        auto checkHashType = [&](const std::optional<Hash> & hash)
+        {
+            if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256))
+                throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true));
+        };
+
+        if (auto rev = input.getRev())
+            checkHashType(rev);
+
+        RepoInfo repoInfo {
+            .shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false),
+            .submodules = getSubmodulesAttr(input),
+            .allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false)
+        };
+
+        repoInfo.cacheType = "git";
+        if (repoInfo.shallow) repoInfo.cacheType += "-shallow";
+        if (repoInfo.submodules) repoInfo.cacheType += "-submodules";
+        if (repoInfo.allRefs) repoInfo.cacheType += "-all-refs";
+
         // file:// URIs are normally not cloned (but otherwise treated the
         // same as remote URIs, i.e. we don't use the working tree or
         // HEAD). Exception: If _NIX_FORCE_HTTP is set, or the repo is a bare git
@@ -393,153 +334,142 @@ struct GitInputScheme : InputScheme
         static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
         auto url = parseURL(getStrAttr(input.attrs, "url"));
         bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git");
-        bool isLocal = url.scheme == "file" && !forceHttp && !isBareRepository;
-        return {isLocal, isLocal ? url.path : url.base};
+        repoInfo.isLocal = url.scheme == "file" && !forceHttp && !isBareRepository;
+        repoInfo.url = repoInfo.isLocal ? url.path : url.base;
+
+        // If this is a local directory and no ref or revision is
+        // given, then allow the use of an unclean working tree.
+        if (!input.getRef() && !input.getRev() && repoInfo.isLocal)
+            repoInfo.workdirInfo = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirInfo();
+
+        return repoInfo;
     }
 
-    std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
+    uint64_t getLastModified(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const
     {
-        Input input(_input);
-        auto gitDir = ".git";
+        Attrs key{{"_what", "gitLastModified"}, {"rev", rev.gitRev()}};
+
+        auto cache = getCache();
+
+        if (auto res = cache->lookup(key))
+            return getIntAttr(*res, "lastModified");
+
+        auto lastModified = GitRepo::openRepo(CanonPath(repoDir))->getLastModified(rev);
+
+        cache->upsert(key, Attrs{{"lastModified", lastModified}});
+
+        return lastModified;
+    }
+
+    uint64_t getRevCount(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const
+    {
+        Attrs key{{"_what", "gitRevCount"}, {"rev", rev.gitRev()}};
+
+        auto cache = getCache();
+
+        if (auto revCountAttrs = cache->lookup(key))
+            return getIntAttr(*revCountAttrs, "revCount");
+
+        Activity act(*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.url));
+
+        auto revCount = GitRepo::openRepo(CanonPath(repoDir))->getRevCount(rev);
+
+        cache->upsert(key, Attrs{{"revCount", revCount}});
+
+        return revCount;
+    }
+
+    std::string getDefaultRef(const RepoInfo & repoInfo) const
+    {
+        auto head = repoInfo.isLocal
+            ? GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirRef()
+            : readHeadCached(repoInfo.url);
+        if (!head) {
+            warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.url);
+            return "master";
+        }
+        return *head;
+    }
+
+    static MakeNotAllowedError makeNotAllowedError(std::string url)
+    {
+        return [url{std::move(url)}](const CanonPath & path) -> RestrictedPathError
+        {
+            if (nix::pathExists(path.abs()))
+                return RestrictedPathError("access to path '%s' is forbidden because it is not under Git control; maybe you should 'git add' it to the repository '%s'?", path, url);
+            else
+                return RestrictedPathError("path '%s' does not exist in Git repository '%s'", path, url);
+        };
+    }
+
+    std::pair<ref<InputAccessor>, Input> getAccessorFromCommit(
+        ref<Store> store,
+        RepoInfo & repoInfo,
+        Input && input) const
+    {
+        assert(!repoInfo.workdirInfo.isDirty);
+
+        auto origRev = input.getRev();
 
         std::string name = input.getName();
 
-        bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
-        bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
-        bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
-
-        std::string cacheType = "git";
-        if (shallow) cacheType += "-shallow";
-        if (submodules) cacheType += "-submodules";
-        if (allRefs) cacheType += "-all-refs";
-
-        auto checkHashType = [&](const std::optional<Hash> & hash)
-        {
-            if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256))
-                throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true));
-        };
-
-        auto getLockedAttrs = [&]()
-        {
-            checkHashType(input.getRev());
-
-            return Attrs({
-                {"type", cacheType},
-                {"name", name},
-                {"rev", input.getRev()->gitRev()},
-            });
-        };
-
-        auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-            -> std::pair<StorePath, Input>
+        auto makeResult2 = [&](const Attrs & infoAttrs, ref<InputAccessor> accessor) -> std::pair<ref<InputAccessor>, Input>
         {
             assert(input.getRev());
-            assert(!_input.getRev() || _input.getRev() == input.getRev());
-            if (!shallow)
+            assert(!origRev || origRev == input.getRev());
+            if (!repoInfo.shallow)
                 input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
             input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
-            return {std::move(storePath), input};
+
+            return {accessor, std::move(input)};
         };
 
-        if (input.getRev()) {
-            if (auto res = getCache()->lookup(store, getLockedAttrs()))
-                return makeResult(res->first, std::move(res->second));
-        }
+        auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> std::pair<ref<InputAccessor>, Input>
+        {
+            // FIXME: remove?
+            //input.attrs.erase("narHash");
+            auto narHash = store->queryPathInfo(storePath)->narHash;
+            input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
 
-        auto [isLocal, actualUrl_] = getActualUrl(input);
-        auto actualUrl = actualUrl_; // work around clang bug
+            auto accessor = makeStorePathAccessor(store, storePath, makeNotAllowedError(repoInfo.url));
 
-        /* If this is a local directory and no ref or revision is given,
-           allow fetching directly from a dirty workdir. */
-        if (!input.getRef() && !input.getRev() && isLocal) {
-            auto workdirInfo = getWorkdirInfo(input, actualUrl);
-            if (!workdirInfo.clean) {
-                return fetchFromWorkdir(store, input, actualUrl, workdirInfo);
-            }
-        }
+            return makeResult2(infoAttrs, accessor);
+        };
 
-        Attrs unlockedAttrs({
-            {"type", cacheType},
-            {"name", name},
-            {"url", actualUrl},
-        });
+        auto originalRef = input.getRef();
+        auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo);
+        input.attrs.insert_or_assign("ref", ref);
 
         Path repoDir;
 
-        if (isLocal) {
-            if (!input.getRef()) {
-                auto head = readHead(actualUrl);
-                if (!head) {
-                    warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl);
-                    head = "master";
-                }
-                input.attrs.insert_or_assign("ref", *head);
-                unlockedAttrs.insert_or_assign("ref", *head);
-            }
-
+        if (repoInfo.isLocal) {
+            repoDir = repoInfo.url;
             if (!input.getRev())
-                input.attrs.insert_or_assign("rev",
-                    Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", *input.getRef() })), htSHA1).gitRev());
-
-            repoDir = actualUrl;
+                input.attrs.insert_or_assign("rev", GitRepo::openRepo(CanonPath(repoDir))->resolveRef(ref).gitRev());
         } else {
-            const bool useHeadRef = !input.getRef();
-            if (useHeadRef) {
-                auto head = readHeadCached(actualUrl);
-                if (!head) {
-                    warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl);
-                    head = "master";
-                }
-                input.attrs.insert_or_assign("ref", *head);
-                unlockedAttrs.insert_or_assign("ref", *head);
-            } else {
-                if (!input.getRev()) {
-                    unlockedAttrs.insert_or_assign("ref", input.getRef().value());
-                }
-            }
-
-            if (auto res = getCache()->lookup(store, unlockedAttrs)) {
-                auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
-                if (!input.getRev() || input.getRev() == rev2) {
-                    input.attrs.insert_or_assign("rev", rev2.gitRev());
-                    return makeResult(res->first, std::move(res->second));
-                }
-            }
-
-            Path cacheDir = getCachePath(actualUrl);
+            Path cacheDir = getCachePath(repoInfo.url);
             repoDir = cacheDir;
-            gitDir = ".";
+            repoInfo.gitDir = ".";
 
             createDirs(dirOf(cacheDir));
-            PathLocks cacheDirLock({cacheDir + ".lock"});
+            PathLocks cacheDirLock({cacheDir});
 
-            if (!pathExists(cacheDir)) {
-                runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir });
-            }
+            auto repo = GitRepo::openRepo(CanonPath(cacheDir), true, true);
 
             Path localRefFile =
-                input.getRef()->compare(0, 5, "refs/") == 0
-                ? cacheDir + "/" + *input.getRef()
-                : cacheDir + "/refs/heads/" + *input.getRef();
+                ref.compare(0, 5, "refs/") == 0
+                ? cacheDir + "/" + ref
+                : cacheDir + "/refs/heads/" + ref;
 
             bool doFetch;
             time_t now = time(0);
 
             /* If a rev was specified, we need to fetch if it's not in the
                repo. */
-            if (input.getRev()) {
-                try {
-                    runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "cat-file", "-e", input.getRev()->gitRev() });
-                    doFetch = false;
-                } catch (ExecError & e) {
-                    if (WIFEXITED(e.status)) {
-                        doFetch = true;
-                    } else {
-                        throw;
-                    }
-                }
+            if (auto rev = input.getRev()) {
+                doFetch = !repo->hasObject(*rev);
             } else {
-                if (allRefs) {
+                if (repoInfo.allRefs) {
                     doFetch = true;
                 } else {
                     /* If the local ref is older than ‘tarball-ttl’ seconds, do a
@@ -551,75 +481,80 @@ struct GitInputScheme : InputScheme
             }
 
             if (doFetch) {
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl));
+                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", repoInfo.url));
 
-                // FIXME: git stderr messes up our progress indicator, so
-                // we're using --quiet for now. Should process its stderr.
                 try {
-                    auto ref = input.getRef();
-                    auto fetchRef = allRefs
+                    auto fetchRef = repoInfo.allRefs
                         ? "refs/*"
-                        : ref->compare(0, 5, "refs/") == 0
-                            ? *ref
-                            : ref == "HEAD"
-                                ? *ref
-                                : "refs/heads/" + *ref;
-                    runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }, {}, true);
+                        : ref.compare(0, 5, "refs/") == 0
+                        ? ref
+                        : ref == "HEAD"
+                        ? ref
+                        : "refs/heads/" + ref;
+
+                    repo->fetch(repoInfo.url, fmt("%s:%s", fetchRef, fetchRef));
                 } catch (Error & e) {
                     if (!pathExists(localRefFile)) throw;
-                    warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
+                    logError(e.info());
+                    warn("could not update local clone of Git repository '%s'; continuing with the most recent version", repoInfo.url);
                 }
 
                 if (!touchCacheFile(localRefFile, now))
                     warn("could not update mtime for file '%s': %s", localRefFile, strerror(errno));
-                if (useHeadRef && !storeCachedHead(actualUrl, *input.getRef()))
-                    warn("could not update cached head '%s' for '%s'", *input.getRef(), actualUrl);
+                if (!originalRef && !storeCachedHead(repoInfo.url, ref))
+                    warn("could not update cached head '%s' for '%s'", ref, repoInfo.url);
             }
 
-            if (!input.getRev())
+            if (auto rev = input.getRev()) {
+                if (!repo->hasObject(*rev))
+                    throw Error(
+                        "Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
+                        "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the "
+                        ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD
+                        "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".",
+                        rev->gitRev(),
+                        ref,
+                        repoInfo.url
+                        );
+            } else
                 input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev());
 
             // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder
         }
 
-        bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-parse", "--is-shallow-repository" })) == "true";
+        auto isShallow = GitRepo::openRepo(CanonPath(repoDir))->isShallow();
 
-        if (isShallow && !shallow)
-            throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified.", actualUrl);
+        if (isShallow && !repoInfo.shallow)
+            throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", repoInfo.url);
 
-        // FIXME: check whether rev is an ancestor of ref.
+        // FIXME: check whether rev is an ancestor of ref?
 
-        printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl);
+        auto rev = *input.getRev();
 
-        /* Now that we know the ref, check again whether we have it in
-           the store. */
-        if (auto res = getCache()->lookup(store, getLockedAttrs()))
-            return makeResult(res->first, std::move(res->second));
-
-        Path tmpDir = createTempDir();
-        AutoDelete delTmpDir(tmpDir, true);
-        PathFilter filter = defaultPathFilter;
-
-        auto result = runProgram(RunOptions {
-            .program = "git",
-            .args = { "-C", repoDir, "--git-dir", gitDir, "cat-file", "commit", input.getRev()->gitRev() },
-            .mergeStderrToStdout = true
+        Attrs infoAttrs({
+            {"rev", rev.gitRev()},
+            {"lastModified", getLastModified(repoInfo, repoDir, rev)},
         });
-        if (WEXITSTATUS(result.first) == 128
-            && result.second.find("bad file") != std::string::npos)
-        {
-            throw Error(
-                "Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
-                "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the "
-                ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD
-                "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".",
-                input.getRev()->gitRev(),
-                *input.getRef(),
-                actualUrl
-            );
+
+        if (!repoInfo.shallow)
+            infoAttrs.insert_or_assign("revCount",
+                getRevCount(repoInfo, repoDir, rev));
+
+        printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.url);
+
+        if (!repoInfo.submodules) {
+            auto accessor = GitRepo::openRepo(CanonPath(repoDir))->getAccessor(rev);
+            return makeResult2(infoAttrs, accessor);
         }
 
-        if (submodules) {
+        else {
+            // FIXME: use libgit2
+            Path tmpDir = createTempDir();
+            AutoDelete delTmpDir(tmpDir, true);
+            PathFilter filter = defaultPathFilter;
+
+            Activity act(*logger, lvlChatty, actUnknown, fmt("copying Git tree '%s' to the store", input.to_string()));
+
             Path tmpGitDir = createTempDir();
             AutoDelete delTmpGitDir(tmpGitDir, true);
 
@@ -634,77 +569,89 @@ struct GitInputScheme : InputScheme
                         "--update-head-ok", "--", repoDir, "refs/*:refs/*" }, {}, true);
             }
 
-            runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() });
+            runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", rev.gitRev() });
 
             /* Ensure that we use the correct origin for fetching
                submodules. This matters for submodules with relative
                URLs. */
-            if (isLocal) {
-                writeFile(tmpGitDir + "/config", readFile(repoDir + "/" + gitDir + "/config"));
+            if (repoInfo.isLocal) {
+                writeFile(tmpGitDir + "/config", readFile(repoDir + "/" + repoInfo.gitDir + "/config"));
 
                 /* Restore the config.bare setting we may have just
                    copied erroneously from the user's repo. */
                 runProgram("git", true, { "-C", tmpDir, "config", "core.bare", "false" });
             } else
-                runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", actualUrl });
+                runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", repoInfo.url });
 
             /* As an optimisation, copy the modules directory of the
                source repo if it exists. */
-            auto modulesPath = repoDir + "/" + gitDir + "/modules";
+            auto modulesPath = repoDir + "/" + repoInfo.gitDir + "/modules";
             if (pathExists(modulesPath)) {
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("copying submodules of '%s'", actualUrl));
+                Activity act(*logger, lvlTalkative, actUnknown, fmt("copying submodules of '%s'", repoInfo.url));
                 runProgram("cp", true, { "-R", "--", modulesPath, tmpGitDir + "/modules" });
             }
 
             {
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", actualUrl));
+                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", repoInfo.url));
                 runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }, {}, true);
             }
 
             filter = isNotDotGitDirectory;
-        } else {
-            // FIXME: should pipe this, or find some better way to extract a
-            // revision.
-            auto source = sinkToSource([&](Sink & sink) {
-                runProgram2({
-                    .program = "git",
-                    .args = { "-C", repoDir, "--git-dir", gitDir, "archive", input.getRev()->gitRev() },
-                    .standardOut = &sink
-                });
-            });
 
-            unpackTarfile(*source, tmpDir);
+            auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
+
+            return makeResult(infoAttrs, std::move(storePath));
+        }
+    }
+
+    std::pair<ref<InputAccessor>, Input> getAccessorFromWorkdir(
+        RepoInfo & repoInfo,
+        Input && input) const
+    {
+        if (!repoInfo.workdirInfo.isDirty) {
+            if (auto ref = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirRef())
+                input.attrs.insert_or_assign("ref", *ref);
+
+            auto rev = repoInfo.workdirInfo.headRev.value();
+
+            input.attrs.insert_or_assign("rev", rev.gitRev());
+
+            input.attrs.insert_or_assign("revCount", getRevCount(repoInfo, repoInfo.url, rev));
+        } else {
+            repoInfo.warnDirty();
+
+            if (repoInfo.workdirInfo.headRev) {
+                input.attrs.insert_or_assign("dirtyRev",
+                    repoInfo.workdirInfo.headRev->gitRev() + "-dirty");
+                input.attrs.insert_or_assign("dirtyShortRev",
+                    repoInfo.workdirInfo.headRev->gitShortRev() + "-dirty");
+            }
         }
 
-        auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
+        input.attrs.insert_or_assign(
+            "lastModified",
+            repoInfo.workdirInfo.headRev
+            ? getLastModified(repoInfo, repoInfo.url, *repoInfo.workdirInfo.headRev)
+            : 0);
 
-        auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() }));
+        input.locked = true; // FIXME
 
-        Attrs infoAttrs({
-            {"rev", input.getRev()->gitRev()},
-            {"lastModified", lastModified},
-        });
+        return {
+            makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url)),
+            std::move(input)
+        };
+    }
 
-        if (!shallow)
-            infoAttrs.insert_or_assign("revCount",
-                std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-list", "--count", input.getRev()->gitRev() })));
+    std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
+    {
+        Input input(_input);
 
-        if (!_input.getRev())
-            getCache()->add(
-                store,
-                unlockedAttrs,
-                infoAttrs,
-                storePath,
-                false);
+        auto repoInfo = getRepoInfo(input);
 
-        getCache()->add(
-            store,
-            getLockedAttrs(),
-            infoAttrs,
-            storePath,
-            true);
-
-        return makeResult(infoAttrs, std::move(storePath));
+        if (input.getRef() || input.getRev() || !repoInfo.isLocal)
+            return getAccessorFromCommit(store, repoInfo, std::move(input));
+        else
+            return getAccessorFromWorkdir(repoInfo, std::move(input));
     }
 };
 
diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk
index 2e8869d83..f21651d77 100644
--- a/src/libfetchers/local.mk
+++ b/src/libfetchers/local.mk
@@ -8,6 +8,6 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc)
 
 libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
 
-libfetchers_LDFLAGS += -pthread
+libfetchers_LDFLAGS += -pthread -lgit2 -larchive
 
 libfetchers_LIBS = libutil libstore
diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh
index fc89f2040..c38cd27eb 100644
--- a/tests/functional/fetchGit.sh
+++ b/tests/functional/fetchGit.sh
@@ -185,11 +185,7 @@ path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref =
 # Nuke the cache
 rm -rf $TEST_HOME/.cache/nix
 
-# Try again, but without 'git' on PATH. This should fail.
-NIX=$(command -v nix)
-(! PATH= $NIX eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" )
-
-# Try again, with 'git' available.  This should work.
+# Try again. This should work.
 path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath")
 [[ $path3 = $path5 ]]
 
@@ -241,6 +237,7 @@ rm -rf $repo/.git
 
 # should succeed for a repo without commits
 git init $repo
+git -C $repo add hello # need to add at least one file to cause the root of the repo to be visible
 path10=$(nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath")
 
 # should succeed for a path with a space
diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh
index 21a4b52de..6e24a80c1 100644
--- a/tests/functional/flakes/flake-in-submodule.sh
+++ b/tests/functional/flakes/flake-in-submodule.sh
@@ -46,7 +46,8 @@ echo '"expression in root repo"' > $rootRepo/root.nix
 git -C $rootRepo add root.nix
 git -C $rootRepo commit -m "Add root.nix"
 
+# FIXME
 # Flake can live inside a submodule and can be accessed via ?dir=submodule
-[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#sub ) = '"expression in submodule"' ]]
+#[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#sub ) = '"expression in submodule"' ]]
 # The flake can access content outside of the submodule
-[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#root ) = '"expression in root repo"' ]]
+#[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#root ) = '"expression in root repo"' ]]

From d88106df24869104cc6c29c726ddfbbfda9dae10 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 27 Oct 2023 18:39:00 +0200
Subject: [PATCH 009/421] Git fetcher: Improve submodule handling

Instead of making a complete copy of the repo, fetching the
submodules, and writing the result to the store (which is all
superexpensive), we now fetch the submodules recursively using the Git
fetcher, and return a union accessor that "mounts" the accessors for
the submodules on top of the root accessor.
---
 src/libfetchers/git-utils.cc            |  78 +++++++++++++++++
 src/libfetchers/git-utils.hh            |  12 +++
 src/libfetchers/git.cc                  | 109 +++++++-----------------
 src/libfetchers/union-input-accessor.cc |  80 +++++++++++++++++
 src/libfetchers/union-input-accessor.hh |   9 ++
 tests/functional/fetchGitSubmodules.sh  |   8 --
 6 files changed, 212 insertions(+), 84 deletions(-)
 create mode 100644 src/libfetchers/union-input-accessor.cc
 create mode 100644 src/libfetchers/union-input-accessor.hh

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 68e39580f..5e3e6dae4 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -1,11 +1,13 @@
 #include "git-utils.hh"
 #include "input-accessor.hh"
 #include "cache.hh"
+#include "finally.hh"
 
 #include <boost/core/span.hpp>
 
 #include <git2/blob.h>
 #include <git2/commit.h>
+#include <git2/config.h>
 #include <git2/describe.h>
 #include <git2/errors.h>
 #include <git2/global.h>
@@ -14,6 +16,7 @@
 #include <git2/remote.h>
 #include <git2/repository.h>
 #include <git2/status.h>
+#include <git2/submodule.h>
 #include <git2/tree.h>
 
 #include <unordered_set>
@@ -63,6 +66,8 @@ typedef std::unique_ptr<git_reference, Deleter<git_reference_free>> Reference;
 typedef std::unique_ptr<git_describe_result, Deleter<git_describe_result_free>> DescribeResult;
 typedef std::unique_ptr<git_status_list, Deleter<git_status_list_free>> StatusList;
 typedef std::unique_ptr<git_remote, Deleter<git_remote_free>> Remote;
+typedef std::unique_ptr<git_config, Deleter<git_config_free>> GitConfig;
+typedef std::unique_ptr<git_config_iterator, Deleter<git_config_iterator_free>> ConfigIterator;
 
 // A helper to ensure that we don't leak objects returned by libgit2.
 template<typename T>
@@ -256,6 +261,17 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         return std::nullopt;
     }
 
+    std::vector<Submodule> getSubmodules(const Hash & rev) override;
+
+    std::string resolveSubmoduleUrl(const std::string & url) override
+    {
+        git_buf buf = GIT_BUF_INIT;
+        if (git_submodule_resolve_url(&buf, *this, url.c_str()))
+            throw Error("resolving Git submodule URL '%s'", url);
+        Finally cleanup = [&]() { git_buf_dispose(&buf); };
+        return buf.ptr;
+    }
+
     bool hasObject(const Hash & oid_) override
     {
         auto oid = hashToOID(oid_);
@@ -400,6 +416,16 @@ struct GitInputAccessor : InputAccessor
         return readBlob(path, true);
     }
 
+    Hash getSubmoduleRev(const CanonPath & path)
+    {
+        auto entry = need(path);
+
+        if (git_tree_entry_type(entry) != GIT_OBJECT_COMMIT)
+            throw Error("'%s' is not a submodule", showPath(path));
+
+        return toHash(*git_tree_entry_id(entry));
+    }
+
     std::map<CanonPath, TreeEntry> lookupCache;
 
     /* Recursively look up 'path' relative to the root. */
@@ -495,4 +521,56 @@ ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev)
     return make_ref<GitInputAccessor>(ref<GitRepoImpl>(shared_from_this()), rev);
 }
 
+std::vector<GitRepoImpl::Submodule> GitRepoImpl::getSubmodules(const Hash & rev)
+{
+    /* Read the .gitmodules files from this revision. */
+    CanonPath modulesFile(".gitmodules");
+
+    auto accessor = getAccessor(rev);
+    if (!accessor->pathExists(modulesFile)) return {};
+
+    /* Parse it. */
+    auto configS = accessor->readFile(modulesFile);
+
+    auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules");
+    writeFull(fdTemp.get(), configS);
+
+    GitConfig config;
+    if (git_config_open_ondisk(Setter(config), pathTemp.c_str()))
+        throw Error("parsing .gitmodules file: %s", git_error_last()->message);
+
+    ConfigIterator it;
+    if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$"))
+        throw Error("iterating over .gitmodules: %s", git_error_last()->message);
+
+    std::map<std::string, std::string> entries;
+
+    while (true) {
+        git_config_entry * entry = nullptr;
+        if (auto err = git_config_next(&entry, it.get())) {
+            if (err == GIT_ITEROVER) break;
+            throw Error("iterating over .gitmodules: %s", git_error_last()->message);
+        }
+        entries.emplace(entry->name + 10, entry->value);
+    }
+
+    std::vector<Submodule> result;
+
+    for (auto & [key, value] : entries) {
+        if (!hasSuffix(key, ".path")) continue;
+        std::string key2(key, 0, key.size() - 5);
+        auto path = CanonPath(value);
+        auto rev = accessor.dynamic_pointer_cast<GitInputAccessor>()->getSubmoduleRev(path);
+        result.push_back(Submodule {
+            .path = path,
+            .url = entries[key2 + ".url"],
+            .branch = entries[key2 + ".branch"],
+            .rev = rev,
+        });
+    }
+
+    return result;
+}
+
+
 }
diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh
index dd2c06672..55e7ef969 100644
--- a/src/libfetchers/git-utils.hh
+++ b/src/libfetchers/git-utils.hh
@@ -38,6 +38,18 @@ struct GitRepo
     /* Get the ref that HEAD points to. */
     virtual std::optional<std::string> getWorkdirRef() = 0;
 
+    struct Submodule
+    {
+        CanonPath path;
+        std::string url;
+        std::string branch;
+        Hash rev;
+    };
+
+    virtual std::vector<Submodule> getSubmodules(const Hash & rev) = 0;
+
+    virtual std::string resolveSubmoduleUrl(const std::string & url) = 0;
+
     struct TarballInfo
     {
         Hash treeHash;
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 55d3a8ebe..42b4aa23a 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -8,6 +8,7 @@
 #include "util.hh"
 #include "git.hh"
 #include "fs-input-accessor.hh"
+#include "union-input-accessor.hh"
 #include "git-utils.hh"
 
 #include "fetch-settings.hh"
@@ -134,11 +135,6 @@ std::optional<std::string> readHeadCached(const std::string & actualUrl)
     return std::nullopt;
 }
 
-bool isNotDotGitDirectory(const Path & path)
-{
-    return baseNameOf(path) != ".git";
-}
-
 }  // end namespace
 
 struct GitInputScheme : InputScheme
@@ -413,7 +409,7 @@ struct GitInputScheme : InputScheme
 
         std::string name = input.getName();
 
-        auto makeResult2 = [&](const Attrs & infoAttrs, ref<InputAccessor> accessor) -> std::pair<ref<InputAccessor>, Input>
+        auto makeResult = [&](const Attrs & infoAttrs, ref<InputAccessor> accessor) -> std::pair<ref<InputAccessor>, Input>
         {
             assert(input.getRev());
             assert(!origRev || origRev == input.getRev());
@@ -424,18 +420,6 @@ struct GitInputScheme : InputScheme
             return {accessor, std::move(input)};
         };
 
-        auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> std::pair<ref<InputAccessor>, Input>
-        {
-            // FIXME: remove?
-            //input.attrs.erase("narHash");
-            auto narHash = store->queryPathInfo(storePath)->narHash;
-            input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
-
-            auto accessor = makeStorePathAccessor(store, storePath, makeNotAllowedError(repoInfo.url));
-
-            return makeResult2(infoAttrs, accessor);
-        };
-
         auto originalRef = input.getRef();
         auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo);
         input.attrs.insert_or_assign("ref", ref);
@@ -542,66 +526,39 @@ struct GitInputScheme : InputScheme
 
         printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.url);
 
-        if (!repoInfo.submodules) {
-            auto accessor = GitRepo::openRepo(CanonPath(repoDir))->getAccessor(rev);
-            return makeResult2(infoAttrs, accessor);
+        auto repo = GitRepo::openRepo(CanonPath(repoDir));
+
+        auto accessor = repo->getAccessor(rev);
+
+        /* If the repo has submodules, fetch them and return a union
+           input accessor consisting of the accessor for the top-level
+           repo and the accessors for the submodules. */
+        if (repoInfo.submodules) {
+            std::map<CanonPath, nix::ref<InputAccessor>> mounts;
+
+            for (auto & submodule : repo->getSubmodules(rev)) {
+                auto resolved = repo->resolveSubmoduleUrl(submodule.url);
+                debug("Git submodule %s: %s %s %s -> %s",
+                    submodule.path, submodule.url, submodule.branch, submodule.rev.gitRev(), resolved);
+                fetchers::Attrs attrs;
+                attrs.insert_or_assign("type", "git");
+                attrs.insert_or_assign("url", resolved);
+                if (submodule.branch != "")
+                    attrs.insert_or_assign("ref", submodule.branch);
+                attrs.insert_or_assign("rev", submodule.rev.gitRev());
+                auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
+                auto [submoduleAccessor, submoduleInput2] =
+                    submoduleInput.scheme->getAccessor(store, submoduleInput);
+                mounts.insert_or_assign(submodule.path, submoduleAccessor);
+            }
+
+            if (!mounts.empty()) {
+                mounts.insert_or_assign(CanonPath::root, accessor);
+                accessor = makeUnionInputAccessor(std::move(mounts));
+            }
         }
 
-        else {
-            // FIXME: use libgit2
-            Path tmpDir = createTempDir();
-            AutoDelete delTmpDir(tmpDir, true);
-            PathFilter filter = defaultPathFilter;
-
-            Activity act(*logger, lvlChatty, actUnknown, fmt("copying Git tree '%s' to the store", input.to_string()));
-
-            Path tmpGitDir = createTempDir();
-            AutoDelete delTmpGitDir(tmpGitDir, true);
-
-            runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", tmpDir, "--separate-git-dir", tmpGitDir });
-
-            {
-                // TODO: repoDir might lack the ref (it only checks if rev
-                // exists, see FIXME above) so use a big hammer and fetch
-                // everything to ensure we get the rev.
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", repoDir));
-                runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
-                        "--update-head-ok", "--", repoDir, "refs/*:refs/*" }, {}, true);
-            }
-
-            runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", rev.gitRev() });
-
-            /* Ensure that we use the correct origin for fetching
-               submodules. This matters for submodules with relative
-               URLs. */
-            if (repoInfo.isLocal) {
-                writeFile(tmpGitDir + "/config", readFile(repoDir + "/" + repoInfo.gitDir + "/config"));
-
-                /* Restore the config.bare setting we may have just
-                   copied erroneously from the user's repo. */
-                runProgram("git", true, { "-C", tmpDir, "config", "core.bare", "false" });
-            } else
-                runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", repoInfo.url });
-
-            /* As an optimisation, copy the modules directory of the
-               source repo if it exists. */
-            auto modulesPath = repoDir + "/" + repoInfo.gitDir + "/modules";
-            if (pathExists(modulesPath)) {
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("copying submodules of '%s'", repoInfo.url));
-                runProgram("cp", true, { "-R", "--", modulesPath, tmpGitDir + "/modules" });
-            }
-
-            {
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", repoInfo.url));
-                runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }, {}, true);
-            }
-
-            filter = isNotDotGitDirectory;
-
-            auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
-
-            return makeResult(infoAttrs, std::move(storePath));
-        }
+        return makeResult(infoAttrs, accessor);
     }
 
     std::pair<ref<InputAccessor>, Input> getAccessorFromWorkdir(
diff --git a/src/libfetchers/union-input-accessor.cc b/src/libfetchers/union-input-accessor.cc
new file mode 100644
index 000000000..940c0e06c
--- /dev/null
+++ b/src/libfetchers/union-input-accessor.cc
@@ -0,0 +1,80 @@
+#include "union-input-accessor.hh"
+
+namespace nix {
+
+struct UnionInputAccessor : InputAccessor
+{
+    std::map<CanonPath, ref<InputAccessor>> mounts;
+
+    UnionInputAccessor(std::map<CanonPath, ref<InputAccessor>> _mounts)
+        : mounts(std::move(_mounts))
+    {
+        // Currently we require a root filesystem. This could be relaxed.
+        assert(mounts.contains(CanonPath::root));
+
+        // FIXME: should check that every mount point exists. Or we
+        // could return dummy parent directories automatically.
+    }
+
+    std::string readFile(const CanonPath & path) override
+    {
+        auto [accessor, subpath] = resolve(path);
+        return accessor->readFile(subpath);
+    }
+
+    bool pathExists(const CanonPath & path) override
+    {
+        auto [accessor, subpath] = resolve(path);
+        return accessor->pathExists(subpath);
+    }
+
+    Stat lstat(const CanonPath & path) override
+    {
+        auto [accessor, subpath] = resolve(path);
+        return accessor->lstat(subpath);
+    }
+
+    DirEntries readDirectory(const CanonPath & path) override
+    {
+        auto [accessor, subpath] = resolve(path);
+        return accessor->readDirectory(subpath);
+    }
+
+    std::string readLink(const CanonPath & path) override
+    {
+        auto [accessor, subpath] = resolve(path);
+        return accessor->readLink(subpath);
+    }
+
+    std::string showPath(const CanonPath & path) override
+    {
+        auto [accessor, subpath] = resolve(path);
+        return accessor->showPath(subpath);
+    }
+
+    std::pair<ref<InputAccessor>, CanonPath> resolve(CanonPath path)
+    {
+        // Find the nearest parent of `path` that is a mount point.
+        std::vector<std::string> ss;
+        while (true) {
+            auto i = mounts.find(path);
+            if (i != mounts.end()) {
+                auto subpath = CanonPath::root;
+                for (auto j = ss.rbegin(); j != ss.rend(); ++j)
+                    subpath.push(*j);
+                return {i->second, std::move(subpath)};
+            }
+
+            assert(!path.isRoot());
+            ss.push_back(std::string(*path.baseName()));
+            path.pop();
+        }
+    }
+};
+
+ref<InputAccessor> makeUnionInputAccessor(std::map<CanonPath, ref<InputAccessor>> mounts)
+{
+    return make_ref<UnionInputAccessor>(std::move(mounts));
+}
+
+}
diff --git a/src/libfetchers/union-input-accessor.hh b/src/libfetchers/union-input-accessor.hh
new file mode 100644
index 000000000..6a1649c1d
--- /dev/null
+++ b/src/libfetchers/union-input-accessor.hh
@@ -0,0 +1,9 @@
+#pragma once
+
+#include "input-accessor.hh"
+
+namespace nix {
+
+ref<InputAccessor> makeUnionInputAccessor(std::map<CanonPath, ref<InputAccessor>> mounts);
+
+}
diff --git a/tests/functional/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh
index df81232e5..369cdc5db 100644
--- a/tests/functional/fetchGitSubmodules.sh
+++ b/tests/functional/fetchGitSubmodules.sh
@@ -118,11 +118,3 @@ cloneRepo=$TEST_ROOT/a/b/gitSubmodulesClone # NB /a/b to make the relative path
 git clone $rootRepo $cloneRepo
 pathIndirect=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath")
 [[ $pathIndirect = $pathWithRelative ]]
-
-# Test that if the clone has the submodule already, we're not fetching
-# it again.
-git -C $cloneRepo submodule update --init
-rm $TEST_HOME/.cache/nix/fetcher-cache*
-rm -rf $subRepo
-pathSubmoduleGone=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath")
-[[ $pathSubmoduleGone = $pathWithRelative ]]

From 669b074f51c4fea6b362313f47eebb4a67f0e89d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sat, 28 Oct 2023 16:16:20 +0200
Subject: [PATCH 010/421] Cleanup

---
 src/libfetchers/git.cc | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 42b4aa23a..a66a51cca 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -409,17 +409,6 @@ struct GitInputScheme : InputScheme
 
         std::string name = input.getName();
 
-        auto makeResult = [&](const Attrs & infoAttrs, ref<InputAccessor> accessor) -> std::pair<ref<InputAccessor>, Input>
-        {
-            assert(input.getRev());
-            assert(!origRev || origRev == input.getRev());
-            if (!repoInfo.shallow)
-                input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
-            input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
-
-            return {accessor, std::move(input)};
-        };
-
         auto originalRef = input.getRef();
         auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo);
         input.attrs.insert_or_assign("ref", ref);
@@ -558,7 +547,13 @@ struct GitInputScheme : InputScheme
             }
         }
 
-        return makeResult(infoAttrs, accessor);
+        assert(input.getRev());
+        assert(!origRev || origRev == input.getRev());
+        if (!repoInfo.shallow)
+            input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
+        input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
+
+        return {accessor, std::move(input)};
     }
 
     std::pair<ref<InputAccessor>, Input> getAccessorFromWorkdir(

From 0c5eac9c4550a6de2cd829d25e628f779e2a29c7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 31 Oct 2023 15:59:25 +0100
Subject: [PATCH 011/421] Git fetcher: Handle submodules for workdirs

---
 src/libfetchers/git-utils.cc                  | 83 +++++++++++--------
 src/libfetchers/git-utils.hh                  | 27 ++++--
 src/libfetchers/git.cc                        | 49 +++++++++--
 tests/functional/flakes/flake-in-submodule.sh | 14 +++-
 4 files changed, 119 insertions(+), 54 deletions(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 5e3e6dae4..5b14cfdb1 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -216,6 +216,43 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         return toHash(*oid);
     }
 
+    std::vector<Submodule> parseSubmodules(const CanonPath & configFile)
+    {
+        GitConfig config;
+        if (git_config_open_ondisk(Setter(config), configFile.abs().c_str()))
+            throw Error("parsing .gitmodules file: %s", git_error_last()->message);
+
+        ConfigIterator it;
+        if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$"))
+            throw Error("iterating over .gitmodules: %s", git_error_last()->message);
+
+        std::map<std::string, std::string> entries;
+
+        while (true) {
+            git_config_entry * entry = nullptr;
+            if (auto err = git_config_next(&entry, it.get())) {
+                if (err == GIT_ITEROVER) break;
+                throw Error("iterating over .gitmodules: %s", git_error_last()->message);
+            }
+            entries.emplace(entry->name + 10, entry->value);
+        }
+
+        std::vector<Submodule> result;
+
+        for (auto & [key, value] : entries) {
+            if (!hasSuffix(key, ".path")) continue;
+            std::string key2(key, 0, key.size() - 5);
+            auto path = CanonPath(value);
+            result.push_back(Submodule {
+                .path = path,
+                .url = entries[key2 + ".url"],
+                .branch = entries[key2 + ".branch"],
+            });
+        }
+
+        return result;
+    }
+
     WorkdirInfo getWorkdirInfo() override
     {
         WorkdirInfo info;
@@ -246,6 +283,11 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         if (git_status_foreach_ext(*this, &options, &statusCallbackTrampoline, &statusCallback))
             throw Error("getting working directory status: %s", git_error_last()->message);
 
+        /* Get submodule info. */
+        auto modulesFile = path + ".gitmodules";
+        if (pathExists(modulesFile.abs()))
+            info.submodules = parseSubmodules(modulesFile);
+
         return info;
     }
 
@@ -261,7 +303,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         return std::nullopt;
     }
 
-    std::vector<Submodule> getSubmodules(const Hash & rev) override;
+    std::vector<std::tuple<Submodule, Hash>> getSubmodules(const Hash & rev) override;
 
     std::string resolveSubmoduleUrl(const std::string & url) override
     {
@@ -521,7 +563,7 @@ ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev)
     return make_ref<GitInputAccessor>(ref<GitRepoImpl>(shared_from_this()), rev);
 }
 
-std::vector<GitRepoImpl::Submodule> GitRepoImpl::getSubmodules(const Hash & rev)
+std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules(const Hash & rev)
 {
     /* Read the .gitmodules files from this revision. */
     CanonPath modulesFile(".gitmodules");
@@ -529,44 +571,17 @@ std::vector<GitRepoImpl::Submodule> GitRepoImpl::getSubmodules(const Hash & rev)
     auto accessor = getAccessor(rev);
     if (!accessor->pathExists(modulesFile)) return {};
 
-    /* Parse it. */
+    /* Parse it and get the revision of each submodule. */
     auto configS = accessor->readFile(modulesFile);
 
     auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules");
     writeFull(fdTemp.get(), configS);
 
-    GitConfig config;
-    if (git_config_open_ondisk(Setter(config), pathTemp.c_str()))
-        throw Error("parsing .gitmodules file: %s", git_error_last()->message);
+    std::vector<std::tuple<Submodule, Hash>> result;
 
-    ConfigIterator it;
-    if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$"))
-        throw Error("iterating over .gitmodules: %s", git_error_last()->message);
-
-    std::map<std::string, std::string> entries;
-
-    while (true) {
-        git_config_entry * entry = nullptr;
-        if (auto err = git_config_next(&entry, it.get())) {
-            if (err == GIT_ITEROVER) break;
-            throw Error("iterating over .gitmodules: %s", git_error_last()->message);
-        }
-        entries.emplace(entry->name + 10, entry->value);
-    }
-
-    std::vector<Submodule> result;
-
-    for (auto & [key, value] : entries) {
-        if (!hasSuffix(key, ".path")) continue;
-        std::string key2(key, 0, key.size() - 5);
-        auto path = CanonPath(value);
-        auto rev = accessor.dynamic_pointer_cast<GitInputAccessor>()->getSubmoduleRev(path);
-        result.push_back(Submodule {
-            .path = path,
-            .url = entries[key2 + ".url"],
-            .branch = entries[key2 + ".branch"],
-            .rev = rev,
-        });
+    for (auto & submodule : parseSubmodules(CanonPath(pathTemp))) {
+        auto rev = accessor.dynamic_pointer_cast<GitInputAccessor>()->getSubmoduleRev(submodule.path);
+        result.push_back({std::move(submodule), rev});
     }
 
     return result;
diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh
index 55e7ef969..a425e5814 100644
--- a/src/libfetchers/git-utils.hh
+++ b/src/libfetchers/git-utils.hh
@@ -20,6 +20,16 @@ struct GitRepo
     /* Return the commit hash to which a ref points. */
     virtual Hash resolveRef(std::string ref) = 0;
 
+    /**
+     * Info about a submodule.
+     */
+    struct Submodule
+    {
+        CanonPath path;
+        std::string url;
+        std::string branch;
+    };
+
     struct WorkdirInfo
     {
         bool isDirty = false;
@@ -31,6 +41,9 @@ struct GitRepo
         /* All files in the working directory that are unchanged,
            modified or added, but excluding deleted files. */
         std::set<CanonPath> files;
+
+        /* The submodules listed in .gitmodules of this workdir. */
+        std::vector<Submodule> submodules;
     };
 
     virtual WorkdirInfo getWorkdirInfo() = 0;
@@ -38,15 +51,11 @@ struct GitRepo
     /* Get the ref that HEAD points to. */
     virtual std::optional<std::string> getWorkdirRef() = 0;
 
-    struct Submodule
-    {
-        CanonPath path;
-        std::string url;
-        std::string branch;
-        Hash rev;
-    };
-
-    virtual std::vector<Submodule> getSubmodules(const Hash & rev) = 0;
+    /**
+     * Return the submodules of this repo at the indicated revision,
+     * along with the revision of each submodule.
+     */
+    virtual std::vector<std::tuple<Submodule, Hash>> getSubmodules(const Hash & rev) = 0;
 
     virtual std::string resolveSubmoduleUrl(const std::string & url) = 0;
 
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index a66a51cca..5471eb260 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -525,16 +525,16 @@ struct GitInputScheme : InputScheme
         if (repoInfo.submodules) {
             std::map<CanonPath, nix::ref<InputAccessor>> mounts;
 
-            for (auto & submodule : repo->getSubmodules(rev)) {
+            for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev)) {
                 auto resolved = repo->resolveSubmoduleUrl(submodule.url);
                 debug("Git submodule %s: %s %s %s -> %s",
-                    submodule.path, submodule.url, submodule.branch, submodule.rev.gitRev(), resolved);
+                    submodule.path, submodule.url, submodule.branch, submoduleRev.gitRev(), resolved);
                 fetchers::Attrs attrs;
                 attrs.insert_or_assign("type", "git");
                 attrs.insert_or_assign("url", resolved);
                 if (submodule.branch != "")
                     attrs.insert_or_assign("ref", submodule.branch);
-                attrs.insert_or_assign("rev", submodule.rev.gitRev());
+                attrs.insert_or_assign("rev", submoduleRev.gitRev());
                 auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
                 auto [submoduleAccessor, submoduleInput2] =
                     submoduleInput.scheme->getAccessor(store, submoduleInput);
@@ -557,9 +557,45 @@ struct GitInputScheme : InputScheme
     }
 
     std::pair<ref<InputAccessor>, Input> getAccessorFromWorkdir(
+        ref<Store> store,
         RepoInfo & repoInfo,
         Input && input) const
     {
+        if (repoInfo.submodules)
+            /* Create mountpoints for the submodules. */
+            for (auto & submodule : repoInfo.workdirInfo.submodules)
+                repoInfo.workdirInfo.files.insert(submodule.path);
+
+        ref<InputAccessor> accessor =
+            makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url));
+
+        /* If the repo has submodules, return a union input accessor
+           consisting of the accessor for the top-level repo and the
+           accessors for the submodule workdirs. */
+        if (repoInfo.submodules && !repoInfo.workdirInfo.submodules.empty()) {
+            std::map<CanonPath, nix::ref<InputAccessor>> mounts;
+
+            for (auto & submodule : repoInfo.workdirInfo.submodules) {
+                auto submodulePath = CanonPath(repoInfo.url) + submodule.path;
+                fetchers::Attrs attrs;
+                attrs.insert_or_assign("type", "git");
+                attrs.insert_or_assign("url", submodulePath.abs());
+                auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
+                auto [submoduleAccessor, submoduleInput2] =
+                    submoduleInput.scheme->getAccessor(store, submoduleInput);
+
+                /* If the submodule is dirty, mark this repo dirty as
+                   well. */
+                if (!submoduleInput2.getRev())
+                    repoInfo.workdirInfo.isDirty = true;
+
+                mounts.insert_or_assign(submodule.path, submoduleAccessor);
+            }
+
+            mounts.insert_or_assign(CanonPath::root, accessor);
+            accessor = makeUnionInputAccessor(std::move(mounts));
+        }
+
         if (!repoInfo.workdirInfo.isDirty) {
             if (auto ref = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirRef())
                 input.attrs.insert_or_assign("ref", *ref);
@@ -588,10 +624,7 @@ struct GitInputScheme : InputScheme
 
         input.locked = true; // FIXME
 
-        return {
-            makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url)),
-            std::move(input)
-        };
+        return {accessor, std::move(input)};
     }
 
     std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
@@ -603,7 +636,7 @@ struct GitInputScheme : InputScheme
         if (input.getRef() || input.getRev() || !repoInfo.isLocal)
             return getAccessorFromCommit(store, repoInfo, std::move(input));
         else
-            return getAccessorFromWorkdir(repoInfo, std::move(input));
+            return getAccessorFromWorkdir(store, repoInfo, std::move(input));
     }
 };
 
diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh
index 6e24a80c1..85a4d3389 100644
--- a/tests/functional/flakes/flake-in-submodule.sh
+++ b/tests/functional/flakes/flake-in-submodule.sh
@@ -46,8 +46,16 @@ echo '"expression in root repo"' > $rootRepo/root.nix
 git -C $rootRepo add root.nix
 git -C $rootRepo commit -m "Add root.nix"
 
-# FIXME
+flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule
+
 # Flake can live inside a submodule and can be accessed via ?dir=submodule
-#[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#sub ) = '"expression in submodule"' ]]
+[[ $(nix eval --json $flakeref#sub ) = '"expression in submodule"' ]]
+
 # The flake can access content outside of the submodule
-#[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#root ) = '"expression in root repo"' ]]
+[[ $(nix eval --json $flakeref#root ) = '"expression in root repo"' ]]
+
+# Check that dirtying a submodule makes the entire thing dirty.
+[[ $(nix flake metadata --json $flakeref | jq -r .locked.rev) != null ]]
+echo '"foo"' > $rootRepo/submodule/sub.nix
+[[ $(nix eval --json $flakeref#sub ) = '"foo"' ]]
+[[ $(nix flake metadata --json $flakeref | jq -r .locked.rev) = null ]]

From e97ac09abeab44fa3d10eb539f0b3d51f8575798 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 6 Jan 2023 18:06:03 -0500
Subject: [PATCH 012/421] Factor out `StoreDirConfig`

More progress on #5729.
---
 src/libstore/path.cc             |  14 ++--
 src/libstore/store-api.cc        |  24 +++---
 src/libstore/store-api.hh        | 103 +------------------------
 src/libstore/store-dir-config.hh | 126 +++++++++++++++++++++++++++++++
 4 files changed, 148 insertions(+), 119 deletions(-)
 create mode 100644 src/libstore/store-dir-config.hh

diff --git a/src/libstore/path.cc b/src/libstore/path.cc
index ec3e53232..69f6d7356 100644
--- a/src/libstore/path.cc
+++ b/src/libstore/path.cc
@@ -1,4 +1,4 @@
-#include "store-api.hh"
+#include "store-dir-config.hh"
 
 #include <sodium.h>
 
@@ -54,7 +54,7 @@ StorePath StorePath::random(std::string_view name)
     return StorePath(hash, name);
 }
 
-StorePath Store::parseStorePath(std::string_view path) const
+StorePath StoreDirConfig::parseStorePath(std::string_view path) const
 {
     auto p = canonPath(std::string(path));
     if (dirOf(p) != storeDir)
@@ -62,7 +62,7 @@ StorePath Store::parseStorePath(std::string_view path) const
     return StorePath(baseNameOf(p));
 }
 
-std::optional<StorePath> Store::maybeParseStorePath(std::string_view path) const
+std::optional<StorePath> StoreDirConfig::maybeParseStorePath(std::string_view path) const
 {
     try {
         return parseStorePath(path);
@@ -71,24 +71,24 @@ std::optional<StorePath> Store::maybeParseStorePath(std::string_view path) const
     }
 }
 
-bool Store::isStorePath(std::string_view path) const
+bool StoreDirConfig::isStorePath(std::string_view path) const
 {
     return (bool) maybeParseStorePath(path);
 }
 
-StorePathSet Store::parseStorePathSet(const PathSet & paths) const
+StorePathSet StoreDirConfig::parseStorePathSet(const PathSet & paths) const
 {
     StorePathSet res;
     for (auto & i : paths) res.insert(parseStorePath(i));
     return res;
 }
 
-std::string Store::printStorePath(const StorePath & path) const
+std::string StoreDirConfig::printStorePath(const StorePath & path) const
 {
     return (storeDir + "/").append(path.to_string());
 }
 
-PathSet Store::printStorePathSet(const StorePathSet & paths) const
+PathSet StoreDirConfig::printStorePathSet(const StorePathSet & paths) const
 {
     PathSet res;
     for (auto & i : paths) res.insert(printStorePath(i));
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 646b0ec7d..a681bb6cf 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -23,13 +23,13 @@ using json = nlohmann::json;
 namespace nix {
 
 
-bool Store::isInStore(PathView path) const
+bool StoreDirConfig::isInStore(PathView path) const
 {
     return isInDir(path, storeDir);
 }
 
 
-std::pair<StorePath, Path> Store::toStorePath(PathView path) const
+std::pair<StorePath, Path> StoreDirConfig::toStorePath(PathView path) const
 {
     if (!isInStore(path))
         throw Error("path '%1%' is not in the Nix store", path);
@@ -143,7 +143,7 @@ StorePath Store::followLinksToStorePath(std::string_view path) const
 */
 
 
-StorePath Store::makeStorePath(std::string_view type,
+StorePath StoreDirConfig::makeStorePath(std::string_view type,
     std::string_view hash, std::string_view name) const
 {
     /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
@@ -154,14 +154,14 @@ StorePath Store::makeStorePath(std::string_view type,
 }
 
 
-StorePath Store::makeStorePath(std::string_view type,
+StorePath StoreDirConfig::makeStorePath(std::string_view type,
     const Hash & hash, std::string_view name) const
 {
     return makeStorePath(type, hash.to_string(HashFormat::Base16, true), name);
 }
 
 
-StorePath Store::makeOutputPath(std::string_view id,
+StorePath StoreDirConfig::makeOutputPath(std::string_view id,
     const Hash & hash, std::string_view name) const
 {
     return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id));
@@ -172,7 +172,7 @@ StorePath Store::makeOutputPath(std::string_view id,
    hacky, but we can't put them in, say, <s2> (per the grammar above)
    since that would be ambiguous. */
 static std::string makeType(
-    const Store & store,
+    const StoreDirConfig & store,
     std::string && type,
     const StoreReferences & references)
 {
@@ -185,7 +185,7 @@ static std::string makeType(
 }
 
 
-StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
+StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
 {
     if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) {
         return makeStorePath(makeType(*this, "source", info.references), info.hash, name);
@@ -201,7 +201,7 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf
 }
 
 
-StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const
+StorePath StoreDirConfig::makeTextPath(std::string_view name, const TextInfo & info) const
 {
     assert(info.hash.type == htSHA256);
     return makeStorePath(
@@ -214,7 +214,7 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons
 }
 
 
-StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const
+StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const
 {
     // New template
     return std::visit(overloaded {
@@ -228,7 +228,7 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA
 }
 
 
-std::pair<StorePath, Hash> Store::computeStorePathFromDump(
+std::pair<StorePath, Hash> StoreDirConfig::computeStorePathFromDump(
     Source & dump,
     std::string_view name,
     FileIngestionMethod method,
@@ -247,7 +247,7 @@ std::pair<StorePath, Hash> Store::computeStorePathFromDump(
 }
 
 
-StorePath Store::computeStorePathForText(
+StorePath StoreDirConfig::computeStorePathForText(
     std::string_view name,
     std::string_view s,
     const StorePathSet & references) const
@@ -1315,7 +1315,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre
 }
 
 
-std::string Store::showPaths(const StorePathSet & paths)
+std::string StoreDirConfig::showPaths(const StorePathSet & paths)
 {
     std::string s;
     for (auto & i : paths) {
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 6aa317e3d..bee5ec16c 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -14,6 +14,7 @@
 #include "config.hh"
 #include "path-info.hh"
 #include "repair-flag.hh"
+#include "store-dir-config.hh"
 
 #include <nlohmann/json_fwd.hpp>
 #include <atomic>
@@ -64,7 +65,6 @@ MakeError(InvalidPath, Error);
 MakeError(Unsupported, Error);
 MakeError(SubstituteGone, Error);
 MakeError(SubstituterDisabled, Error);
-MakeError(BadStorePath, Error);
 
 MakeError(InvalidStoreURI, Error);
 
@@ -97,11 +97,11 @@ struct KeyedBuildResult;
 
 typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
 
-struct StoreConfig : public Config
+struct StoreConfig : public StoreDirConfig
 {
     typedef std::map<std::string, std::string> Params;
 
-    using Config::Config;
+    using StoreDirConfig::StoreDirConfig;
 
     StoreConfig() = delete;
 
@@ -131,15 +131,6 @@ struct StoreConfig : public Config
         return std::nullopt;
     }
 
-    const PathSetting storeDir_{this, settings.nixStore,
-        "store",
-        R"(
-          Logical location of the Nix store, usually
-          `/nix/store`. Note that you can only copy store paths
-          between stores if they have the same `store` setting.
-        )"};
-    const Path storeDir = storeDir_;
-
     const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size",
         "Size of the in-memory store path metadata cache."};
 
@@ -224,45 +215,6 @@ public:
 
     virtual std::string getUri() = 0;
 
-    StorePath parseStorePath(std::string_view path) const;
-
-    std::optional<StorePath> maybeParseStorePath(std::string_view path) const;
-
-    std::string printStorePath(const StorePath & path) const;
-
-    /**
-     * Deprecated
-     *
-     * \todo remove
-     */
-    StorePathSet parseStorePathSet(const PathSet & paths) const;
-
-    PathSet printStorePathSet(const StorePathSet & path) const;
-
-    /**
-     * Display a set of paths in human-readable form (i.e., between quotes
-     * and separated by commas).
-     */
-    std::string showPaths(const StorePathSet & paths);
-
-    /**
-     * @return true if ‘path’ is in the Nix store (but not the Nix
-     * store itself).
-     */
-    bool isInStore(PathView path) const;
-
-    /**
-     * @return true if ‘path’ is a store path, i.e. a direct child of the
-     * Nix store.
-     */
-    bool isStorePath(std::string_view path) const;
-
-    /**
-     * Split a path like /nix/store/<hash>-<name>/<bla> into
-     * /nix/store/<hash>-<name> and /<bla>.
-     */
-    std::pair<StorePath, Path> toStorePath(PathView path) const;
-
     /**
      * Follow symlinks until we end up with a path in the Nix store.
      */
@@ -274,55 +226,6 @@ public:
      */
     StorePath followLinksToStorePath(std::string_view path) const;
 
-    /**
-     * Constructs a unique store path name.
-     */
-    StorePath makeStorePath(std::string_view type,
-        std::string_view hash, std::string_view name) const;
-    StorePath makeStorePath(std::string_view type,
-        const Hash & hash, std::string_view name) const;
-
-    StorePath makeOutputPath(std::string_view id,
-        const Hash & hash, std::string_view name) const;
-
-    StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const;
-
-    StorePath makeTextPath(std::string_view name, const TextInfo & info) const;
-
-    StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const;
-
-    /**
-     * Read-only variant of addToStoreFromDump(). It returns the store
-     * path to which a NAR or flat file would be written.
-     */
-    std::pair<StorePath, Hash> computeStorePathFromDump(
-        Source & dump,
-        std::string_view name,
-        FileIngestionMethod method = FileIngestionMethod::Recursive,
-        HashType hashAlgo = htSHA256,
-        const StorePathSet & references = {}) const;
-
-    /**
-     * Preparatory part of addTextToStore().
-     *
-     * !!! Computation of the path should take the references given to
-     * addTextToStore() into account, otherwise we have a (relatively
-     * minor) security hole: a caller can register a source file with
-     * bogus references.  If there are too many references, the path may
-     * not be garbage collected when it has to be (not really a problem,
-     * the caller could create a root anyway), or it may be garbage
-     * collected when it shouldn't be (more serious).
-     *
-     * Hashing the references would solve this (bogus references would
-     * simply yield a different store path, so other users wouldn't be
-     * affected), but it has some backwards compatibility issues (the
-     * hashing scheme changes), so I'm not doing that for now.
-     */
-    StorePath computeStorePathForText(
-        std::string_view name,
-        std::string_view s,
-        const StorePathSet & references) const;
-
     /**
      * Check whether a path is valid.
      */
diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh
new file mode 100644
index 000000000..53843d663
--- /dev/null
+++ b/src/libstore/store-dir-config.hh
@@ -0,0 +1,126 @@
+#pragma once
+
+#include "path.hh"
+#include "hash.hh"
+#include "content-address.hh"
+#include "globals.hh"
+#include "config.hh"
+
+#include <map>
+#include <string>
+#include <variant>
+
+
+namespace nix {
+
+MakeError(BadStorePath, Error);
+
+struct StoreDirConfig : public Config
+{
+    using Config::Config;
+
+    StoreDirConfig() = delete;
+
+    virtual ~StoreDirConfig() = default;
+
+    const PathSetting storeDir_{this, settings.nixStore,
+        "store",
+        R"(
+          Logical location of the Nix store, usually
+          `/nix/store`. Note that you can only copy store paths
+          between stores if they have the same `store` setting.
+        )"};
+    const Path storeDir = storeDir_;
+
+    // pure methods
+
+    StorePath parseStorePath(std::string_view path) const;
+
+    std::optional<StorePath> maybeParseStorePath(std::string_view path) const;
+
+    std::string printStorePath(const StorePath & path) const;
+
+    /**
+     * Deprecated
+     *
+     * \todo remove
+     */
+    StorePathSet parseStorePathSet(const PathSet & paths) const;
+
+    PathSet printStorePathSet(const StorePathSet & path) const;
+
+    /**
+     * Display a set of paths in human-readable form (i.e., between quotes
+     * and separated by commas).
+     */
+    std::string showPaths(const StorePathSet & paths);
+
+    /**
+     * @return true if ‘path’ is in the Nix store (but not the Nix
+     * store itself).
+     */
+    bool isInStore(PathView path) const;
+
+    /**
+     * @return true if ‘path’ is a store path, i.e. a direct child of the
+     * Nix store.
+     */
+    bool isStorePath(std::string_view path) const;
+
+    /**
+     * Split a path like /nix/store/<hash>-<name>/<bla> into
+     * /nix/store/<hash>-<name> and /<bla>.
+     */
+    std::pair<StorePath, Path> toStorePath(PathView path) const;
+
+    /**
+     * Constructs a unique store path name.
+     */
+    StorePath makeStorePath(std::string_view type,
+        std::string_view hash, std::string_view name) const;
+    StorePath makeStorePath(std::string_view type,
+        const Hash & hash, std::string_view name) const;
+
+    StorePath makeOutputPath(std::string_view id,
+        const Hash & hash, std::string_view name) const;
+
+    StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const;
+
+    StorePath makeTextPath(std::string_view name, const TextInfo & info) const;
+
+    StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const;
+
+    /**
+     * Read-only variant of addToStoreFromDump(). It returns the store
+     * path to which a NAR or flat file would be written.
+     */
+    std::pair<StorePath, Hash> computeStorePathFromDump(
+        Source & dump,
+        std::string_view name,
+        FileIngestionMethod method = FileIngestionMethod::Recursive,
+        HashType hashAlgo = htSHA256,
+        const StorePathSet & references = {}) const;
+
+    /**
+     * Preparatory part of addTextToStore().
+     *
+     * !!! Computation of the path should take the references given to
+     * addTextToStore() into account, otherwise we have a (relatively
+     * minor) security hole: a caller can register a source file with
+     * bogus references.  If there are too many references, the path may
+     * not be garbage collected when it has to be (not really a problem,
+     * the caller could create a root anyway), or it may be garbage
+     * collected when it shouldn't be (more serious).
+     *
+     * Hashing the references would solve this (bogus references would
+     * simply yield a different store path, so other users wouldn't be
+     * affected), but it has some backwards compatibility issues (the
+     * hashing scheme changes), so I'm not doing that for now.
+     */
+    StorePath computeStorePathForText(
+        std::string_view name,
+        std::string_view s,
+        const StorePathSet & references) const;
+};
+
+}

From dde1d863388617b3a63db808c125f274c86a3222 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 18 Mar 2022 15:35:45 +0000
Subject: [PATCH 013/421] Restrict some code to `StoreDirConfig`

 - part of eval cache
 - part of derivations
 - derived path
 - store path with outputs
 - serializers
---
 perl/lib/Nix/Store.xs                         |  1 +
 src/libcmd/built-path.cc                      |  8 ++--
 src/libcmd/built-path.hh                      | 18 ++++-----
 src/libexpr/eval-cache.cc                     |  6 +--
 src/libexpr/primops/fetchClosure.cc           |  1 +
 src/libexpr/value.hh                          |  1 -
 src/libstore/builtins/buildenv.cc             |  1 +
 src/libstore/builtins/buildenv.hh             |  1 -
 src/libstore/common-protocol-impl.hh          |  4 +-
 src/libstore/common-protocol.cc               | 28 ++++++-------
 src/libstore/common-protocol.hh               |  8 ++--
 src/libstore/derivations.cc                   | 32 +++++++--------
 src/libstore/derivations.hh                   | 26 +++++++------
 src/libstore/derived-path.cc                  | 39 ++++++++++---------
 src/libstore/derived-path.hh                  | 37 ++++++++++--------
 .../length-prefixed-protocol-helper.hh        | 22 +++++------
 src/libstore/local-store.cc                   |  1 +
 src/libstore/misc.cc                          |  1 +
 src/libstore/path-with-outputs.cc             |  4 +-
 src/libstore/path-with-outputs.hh             | 10 +++--
 src/libstore/serve-protocol-impl.hh           |  8 ++--
 src/libstore/serve-protocol.cc                |  4 +-
 src/libstore/serve-protocol.hh                | 12 +++---
 src/libstore/store-api.cc                     |  2 +
 src/libstore/store-api.hh                     | 11 ++++--
 src/libstore/worker-protocol-impl.hh          |  8 ++--
 src/libstore/worker-protocol.cc               | 24 ++++++------
 src/libstore/worker-protocol.hh               | 12 +++---
 src/nix-build/nix-build.cc                    |  1 +
 src/nix-copy-closure/nix-copy-closure.cc      |  1 +
 30 files changed, 175 insertions(+), 157 deletions(-)

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index 08f812b31..210d50b6e 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -9,6 +9,7 @@
 #undef do_close
 
 #include "derivations.hh"
+#include "realisation.hh"
 #include "globals.hh"
 #include "store-api.hh"
 #include "util.hh"
diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc
index 9a2dce806..8e2efc7c3 100644
--- a/src/libcmd/built-path.cc
+++ b/src/libcmd/built-path.cc
@@ -80,7 +80,7 @@ SingleDerivedPath SingleBuiltPath::discardOutputPath() const
     );
 }
 
-nlohmann::json BuiltPath::Built::toJSON(const Store & store) const
+nlohmann::json BuiltPath::Built::toJSON(const StoreDirConfig & store) const
 {
     nlohmann::json res;
     res["drvPath"] = drvPath->toJSON(store);
@@ -90,7 +90,7 @@ nlohmann::json BuiltPath::Built::toJSON(const Store & store) const
     return res;
 }
 
-nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const
+nlohmann::json SingleBuiltPath::Built::toJSON(const StoreDirConfig & store) const
 {
     nlohmann::json res;
     res["drvPath"] = drvPath->toJSON(store);
@@ -100,14 +100,14 @@ nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const
     return res;
 }
 
-nlohmann::json SingleBuiltPath::toJSON(const Store & store) const
+nlohmann::json SingleBuiltPath::toJSON(const StoreDirConfig & store) const
 {
     return std::visit([&](const auto & buildable) {
         return buildable.toJSON(store);
     }, raw());
 }
 
-nlohmann::json BuiltPath::toJSON(const Store & store) const
+nlohmann::json BuiltPath::toJSON(const StoreDirConfig & store) const
 {
     return std::visit([&](const auto & buildable) {
         return buildable.toJSON(store);
diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh
index e677bc810..51918f96c 100644
--- a/src/libcmd/built-path.hh
+++ b/src/libcmd/built-path.hh
@@ -11,9 +11,9 @@ struct SingleBuiltPathBuilt {
 
     SingleDerivedPathBuilt discardOutputPath() const;
 
-    std::string to_string(const Store & store) const;
-    static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
-    nlohmann::json toJSON(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
+    static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view);
+    nlohmann::json toJSON(const StoreDirConfig & store) const;
 
     DECLARE_CMP(SingleBuiltPathBuilt);
 };
@@ -38,8 +38,8 @@ struct SingleBuiltPath : _SingleBuiltPathRaw {
 
     SingleDerivedPath discardOutputPath() const;
 
-    static SingleBuiltPath parse(const Store & store, std::string_view);
-    nlohmann::json toJSON(const Store & store) const;
+    static SingleBuiltPath parse(const StoreDirConfig & store, std::string_view);
+    nlohmann::json toJSON(const StoreDirConfig & store) const;
 };
 
 static inline ref<SingleBuiltPath> staticDrv(StorePath drvPath)
@@ -56,9 +56,9 @@ struct BuiltPathBuilt {
     ref<SingleBuiltPath> drvPath;
     std::map<std::string, StorePath> outputs;
 
-    std::string to_string(const Store & store) const;
-    static BuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
-    nlohmann::json toJSON(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
+    static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view);
+    nlohmann::json toJSON(const StoreDirConfig & store) const;
 
     DECLARE_CMP(BuiltPathBuilt);
 };
@@ -86,7 +86,7 @@ struct BuiltPath : _BuiltPathRaw {
     StorePathSet outPaths() const;
     RealisedPath::Set toRealisedPaths(Store & store) const;
 
-    nlohmann::json toJSON(const Store & store) const;
+    nlohmann::json toJSON(const StoreDirConfig & store) const;
 };
 
 typedef std::vector<BuiltPath> BuiltPaths;
diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc
index 10fc799a9..824f94ba1 100644
--- a/src/libexpr/eval-cache.cc
+++ b/src/libexpr/eval-cache.cc
@@ -21,7 +21,7 @@ struct AttrDb
 {
     std::atomic_bool failed{false};
 
-    const Store & cfg;
+    const StoreDirConfig & cfg;
 
     struct State
     {
@@ -38,7 +38,7 @@ struct AttrDb
     SymbolTable & symbols;
 
     AttrDb(
-        const Store & cfg,
+        const StoreDirConfig & cfg,
         const Hash & fingerprint,
         SymbolTable & symbols)
         : cfg(cfg)
@@ -322,7 +322,7 @@ struct AttrDb
 };
 
 static std::shared_ptr<AttrDb> makeAttrDb(
-    const Store & cfg,
+    const StoreDirConfig & cfg,
     const Hash & fingerprint,
     SymbolTable & symbols)
 {
diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc
index b86ef6b93..27147a5d1 100644
--- a/src/libexpr/primops/fetchClosure.cc
+++ b/src/libexpr/primops/fetchClosure.cc
@@ -1,5 +1,6 @@
 #include "primops.hh"
 #include "store-api.hh"
+#include "realisation.hh"
 #include "make-content-addressed.hh"
 #include "url.hh"
 
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 622e613ea..20f268a3e 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -66,7 +66,6 @@ class Symbol;
 class PosIdx;
 struct Pos;
 class StorePath;
-class Store;
 class EvalState;
 class XMLWriter;
 
diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc
index c8911d153..9283251ac 100644
--- a/src/libstore/builtins/buildenv.cc
+++ b/src/libstore/builtins/buildenv.cc
@@ -1,4 +1,5 @@
 #include "buildenv.hh"
+#include "derivations.hh"
 
 #include <sys/stat.h>
 #include <sys/types.h>
diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh
index 0923c2adb..8bebd390d 100644
--- a/src/libstore/builtins/buildenv.hh
+++ b/src/libstore/builtins/buildenv.hh
@@ -1,7 +1,6 @@
 #pragma once
 ///@file
 
-#include "derivations.hh"
 #include "store-api.hh"
 
 namespace nix {
diff --git a/src/libstore/common-protocol-impl.hh b/src/libstore/common-protocol-impl.hh
index 079c182b8..360882c02 100644
--- a/src/libstore/common-protocol-impl.hh
+++ b/src/libstore/common-protocol-impl.hh
@@ -16,11 +16,11 @@ namespace nix {
 /* protocol-agnostic templates */
 
 #define COMMON_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \
-    TEMPLATE T CommonProto::Serialise< T >::read(const Store & store, CommonProto::ReadConn conn) \
+    TEMPLATE T CommonProto::Serialise< T >::read(const StoreDirConfig & store, CommonProto::ReadConn conn) \
     { \
         return LengthPrefixedProtoHelper<CommonProto, T >::read(store, conn); \
     } \
-    TEMPLATE void CommonProto::Serialise< T >::write(const Store & store, CommonProto::WriteConn conn, const T & t) \
+    TEMPLATE void CommonProto::Serialise< T >::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const T & t) \
     { \
         LengthPrefixedProtoHelper<CommonProto, T >::write(store, conn, t); \
     }
diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc
index f906814bc..c14a6cfcd 100644
--- a/src/libstore/common-protocol.cc
+++ b/src/libstore/common-protocol.cc
@@ -14,40 +14,40 @@ namespace nix {
 
 /* protocol-agnostic definitions */
 
-std::string CommonProto::Serialise<std::string>::read(const Store & store, CommonProto::ReadConn conn)
+std::string CommonProto::Serialise<std::string>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
 {
     return readString(conn.from);
 }
 
-void CommonProto::Serialise<std::string>::write(const Store & store, CommonProto::WriteConn conn, const std::string & str)
+void CommonProto::Serialise<std::string>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::string & str)
 {
     conn.to << str;
 }
 
 
-StorePath CommonProto::Serialise<StorePath>::read(const Store & store, CommonProto::ReadConn conn)
+StorePath CommonProto::Serialise<StorePath>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
 {
     return store.parseStorePath(readString(conn.from));
 }
 
-void CommonProto::Serialise<StorePath>::write(const Store & store, CommonProto::WriteConn conn, const StorePath & storePath)
+void CommonProto::Serialise<StorePath>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const StorePath & storePath)
 {
     conn.to << store.printStorePath(storePath);
 }
 
 
-ContentAddress CommonProto::Serialise<ContentAddress>::read(const Store & store, CommonProto::ReadConn conn)
+ContentAddress CommonProto::Serialise<ContentAddress>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
 {
     return ContentAddress::parse(readString(conn.from));
 }
 
-void CommonProto::Serialise<ContentAddress>::write(const Store & store, CommonProto::WriteConn conn, const ContentAddress & ca)
+void CommonProto::Serialise<ContentAddress>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const ContentAddress & ca)
 {
     conn.to << renderContentAddress(ca);
 }
 
 
-Realisation CommonProto::Serialise<Realisation>::read(const Store & store, CommonProto::ReadConn conn)
+Realisation CommonProto::Serialise<Realisation>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
 {
     std::string rawInput = readString(conn.from);
     return Realisation::fromJSON(
@@ -56,41 +56,41 @@ Realisation CommonProto::Serialise<Realisation>::read(const Store & store, Commo
     );
 }
 
-void CommonProto::Serialise<Realisation>::write(const Store & store, CommonProto::WriteConn conn, const Realisation & realisation)
+void CommonProto::Serialise<Realisation>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const Realisation & realisation)
 {
     conn.to << realisation.toJSON().dump();
 }
 
 
-DrvOutput CommonProto::Serialise<DrvOutput>::read(const Store & store, CommonProto::ReadConn conn)
+DrvOutput CommonProto::Serialise<DrvOutput>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
 {
     return DrvOutput::parse(readString(conn.from));
 }
 
-void CommonProto::Serialise<DrvOutput>::write(const Store & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput)
+void CommonProto::Serialise<DrvOutput>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput)
 {
     conn.to << drvOutput.to_string();
 }
 
 
-std::optional<StorePath> CommonProto::Serialise<std::optional<StorePath>>::read(const Store & store, CommonProto::ReadConn conn)
+std::optional<StorePath> CommonProto::Serialise<std::optional<StorePath>>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
 {
     auto s = readString(conn.from);
     return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
 }
 
-void CommonProto::Serialise<std::optional<StorePath>>::write(const Store & store, CommonProto::WriteConn conn, const std::optional<StorePath> & storePathOpt)
+void CommonProto::Serialise<std::optional<StorePath>>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::optional<StorePath> & storePathOpt)
 {
     conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
 }
 
 
-std::optional<ContentAddress> CommonProto::Serialise<std::optional<ContentAddress>>::read(const Store & store, CommonProto::ReadConn conn)
+std::optional<ContentAddress> CommonProto::Serialise<std::optional<ContentAddress>>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
 {
     return ContentAddress::parseOpt(readString(conn.from));
 }
 
-void CommonProto::Serialise<std::optional<ContentAddress>>::write(const Store & store, CommonProto::WriteConn conn, const std::optional<ContentAddress> & caOpt)
+void CommonProto::Serialise<std::optional<ContentAddress>>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::optional<ContentAddress> & caOpt)
 {
     conn.to << (caOpt ? renderContentAddress(*caOpt) : "");
 }
diff --git a/src/libstore/common-protocol.hh b/src/libstore/common-protocol.hh
index f3f28972a..a878e84c9 100644
--- a/src/libstore/common-protocol.hh
+++ b/src/libstore/common-protocol.hh
@@ -5,7 +5,7 @@
 
 namespace nix {
 
-class Store;
+struct StoreDirConfig;
 struct Source;
 
 // items being serialized
@@ -48,7 +48,7 @@ struct CommonProto
      * infer the type instead of having to write it down explicitly.
      */
     template<typename T>
-    static void write(const Store & store, WriteConn conn, const T & t)
+    static void write(const StoreDirConfig & store, WriteConn conn, const T & t)
     {
         CommonProto::Serialise<T>::write(store, conn, t);
     }
@@ -57,8 +57,8 @@ struct CommonProto
 #define DECLARE_COMMON_SERIALISER(T) \
     struct CommonProto::Serialise< T > \
     { \
-        static T read(const Store & store, CommonProto::ReadConn conn); \
-        static void write(const Store & store, CommonProto::WriteConn conn, const T & str); \
+        static T read(const StoreDirConfig & store, CommonProto::ReadConn conn); \
+        static void write(const StoreDirConfig & store, CommonProto::WriteConn conn, const T & str); \
     }
 
 template<>
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 1fecd1c97..239232c8e 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -11,7 +11,7 @@
 
 namespace nix {
 
-std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, OutputNameView outputName) const
+std::optional<StorePath> DerivationOutput::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const
 {
     return std::visit(overloaded {
         [](const DerivationOutput::InputAddressed & doi) -> std::optional<StorePath> {
@@ -35,7 +35,7 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
 }
 
 
-StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, OutputNameView outputName) const
+StorePath DerivationOutput::CAFixed::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const
 {
     return store.makeFixedOutputPathFromCA(
         outputPathName(drvName, outputName),
@@ -215,7 +215,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
 
 
 static DerivationOutput parseDerivationOutput(
-    const Store & store,
+    const StoreDirConfig & store,
     std::string_view pathS, std::string_view hashAlgo, std::string_view hashS,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -262,7 +262,7 @@ static DerivationOutput parseDerivationOutput(
 }
 
 static DerivationOutput parseDerivationOutput(
-    const Store & store, std::istringstream & str,
+    const StoreDirConfig & store, std::istringstream & str,
     const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings)
 {
     expect(str, ","); const auto pathS = parseString(str);
@@ -291,7 +291,7 @@ enum struct DerivationATermVersion {
 };
 
 static DerivedPathMap<StringSet>::ChildNode parseDerivedPathMapNode(
-    const Store & store,
+    const StoreDirConfig & store,
     std::istringstream & str,
     DerivationATermVersion version)
 {
@@ -338,7 +338,7 @@ static DerivedPathMap<StringSet>::ChildNode parseDerivedPathMapNode(
 
 
 Derivation parseDerivation(
-    const Store & store, std::string && s, std::string_view name,
+    const StoreDirConfig & store, std::string && s, std::string_view name,
     const ExperimentalFeatureSettings & xpSettings)
 {
     Derivation drv;
@@ -471,7 +471,7 @@ static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIt
 }
 
 
-static void unparseDerivedPathMapNode(const Store & store, std::string & s, const DerivedPathMap<StringSet>::ChildNode & node)
+static void unparseDerivedPathMapNode(const StoreDirConfig & store, std::string & s, const DerivedPathMap<StringSet>::ChildNode & node)
 {
     s += ',';
     if (node.childMap.empty()) {
@@ -512,7 +512,7 @@ static bool hasDynamicDrvDep(const Derivation & drv)
 }
 
 
-std::string Derivation::unparse(const Store & store, bool maskOutputs,
+std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs,
     DerivedPathMap<StringSet>::ChildNode::Map * actualInputs) const
 {
     std::string s;
@@ -846,7 +846,7 @@ std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation &
 }
 
 
-static DerivationOutput readDerivationOutput(Source & in, const Store & store)
+static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & store)
 {
     const auto pathS = readString(in);
     const auto hashAlgo = readString(in);
@@ -863,7 +863,7 @@ StringSet BasicDerivation::outputNames() const
     return names;
 }
 
-DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const
+DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const StoreDirConfig & store) const
 {
     DerivationOutputsAndOptPaths outsAndOptPaths;
     for (auto & [outputName, output] : outputs)
@@ -885,7 +885,7 @@ std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath)
 }
 
 
-Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name)
+Source & readDerivation(Source & in, const StoreDirConfig & store, BasicDerivation & drv, std::string_view name)
 {
     drv.name = name;
 
@@ -913,7 +913,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
 }
 
 
-void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
+void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDerivation & drv)
 {
     out << drv.outputs.size();
     for (auto & i : drv.outputs) {
@@ -1154,7 +1154,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
 const Hash impureOutputHash = hashString(htSHA256, "impure");
 
 nlohmann::json DerivationOutput::toJSON(
-    const Store & store, std::string_view drvName, OutputNameView outputName) const
+    const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const
 {
     nlohmann::json res = nlohmann::json::object();
     std::visit(overloaded {
@@ -1181,7 +1181,7 @@ nlohmann::json DerivationOutput::toJSON(
 
 
 DerivationOutput DerivationOutput::fromJSON(
-    const Store & store, std::string_view drvName, OutputNameView outputName,
+    const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName,
     const nlohmann::json & _json,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -1250,7 +1250,7 @@ DerivationOutput DerivationOutput::fromJSON(
 }
 
 
-nlohmann::json Derivation::toJSON(const Store & store) const
+nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const
 {
     nlohmann::json res = nlohmann::json::object();
 
@@ -1303,7 +1303,7 @@ nlohmann::json Derivation::toJSON(const Store & store) const
 
 
 Derivation Derivation::fromJSON(
-    const Store & store,
+    const StoreDirConfig & store,
     const nlohmann::json & json,
     const ExperimentalFeatureSettings & xpSettings)
 {
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index fa14e7536..219e8e7d7 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -17,7 +17,7 @@
 
 namespace nix {
 
-class Store;
+struct StoreDirConfig;
 
 /* Abstract syntax of derivations. */
 
@@ -55,7 +55,7 @@ struct DerivationOutput
          * @param drvName The name of the derivation this is an output of, without the `.drv`.
          * @param outputName The name of this output.
          */
-        StorePath path(const Store & store, std::string_view drvName, OutputNameView outputName) const;
+        StorePath path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const;
 
         GENERATE_CMP(CAFixed, me->ca);
     };
@@ -132,17 +132,17 @@ struct DerivationOutput
      * the safer interface provided by
      * BasicDerivation::outputsAndOptPaths
      */
-    std::optional<StorePath> path(const Store & store, std::string_view drvName, OutputNameView outputName) const;
+    std::optional<StorePath> path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const;
 
     nlohmann::json toJSON(
-        const Store & store,
+        const StoreDirConfig & store,
         std::string_view drvName,
         OutputNameView outputName) const;
     /**
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static DerivationOutput fromJSON(
-        const Store & store,
+        const StoreDirConfig & store,
         std::string_view drvName,
         OutputNameView outputName,
         const nlohmann::json & json,
@@ -304,7 +304,7 @@ struct BasicDerivation
      * augmented with knowledge of the Store paths they would be written
      * into.
      */
-    DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const;
+    DerivationOutputsAndOptPaths outputsAndOptPaths(const StoreDirConfig & store) const;
 
     static std::string_view nameFromPath(const StorePath & storePath);
 
@@ -318,6 +318,8 @@ struct BasicDerivation
         me->name);
 };
 
+class Store;
+
 struct Derivation : BasicDerivation
 {
     /**
@@ -328,7 +330,7 @@ struct Derivation : BasicDerivation
     /**
      * Print a derivation.
      */
-    std::string unparse(const Store & store, bool maskOutputs,
+    std::string unparse(const StoreDirConfig & store, bool maskOutputs,
         DerivedPathMap<StringSet>::ChildNode::Map * actualInputs = nullptr) const;
 
     /**
@@ -365,9 +367,9 @@ struct Derivation : BasicDerivation
     Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
     Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
 
-    nlohmann::json toJSON(const Store & store) const;
+    nlohmann::json toJSON(const StoreDirConfig & store) const;
     static Derivation fromJSON(
-        const Store & store,
+        const StoreDirConfig & store,
         const nlohmann::json & json,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
@@ -391,7 +393,7 @@ StorePath writeDerivation(Store & store,
  * Read a derivation from a file.
  */
 Derivation parseDerivation(
-    const Store & store,
+    const StoreDirConfig & store,
     std::string && s,
     std::string_view name,
     const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
@@ -493,8 +495,8 @@ extern Sync<DrvHashes> drvHashes;
 struct Source;
 struct Sink;
 
-Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name);
-void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv);
+Source & readDerivation(Source & in, const StoreDirConfig & store, BasicDerivation & drv, std::string_view name);
+void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDerivation & drv);
 
 /**
  * This creates an opaque and almost certainly unique string
diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc
index 47d784deb..3105dbc93 100644
--- a/src/libstore/derived-path.cc
+++ b/src/libstore/derived-path.cc
@@ -1,4 +1,5 @@
 #include "derived-path.hh"
+#include "derivations.hh"
 #include "store-api.hh"
 
 #include <nlohmann/json.hpp>
@@ -32,7 +33,7 @@ CMP(SingleDerivedPath, DerivedPathBuilt, outputs)
 #undef CMP
 #undef CMP_ONE
 
-nlohmann::json DerivedPath::Opaque::toJSON(const Store & store) const
+nlohmann::json DerivedPath::Opaque::toJSON(const StoreDirConfig & store) const
 {
     return store.printStorePath(path);
 }
@@ -86,50 +87,50 @@ nlohmann::json DerivedPath::toJSON(Store & store) const
     }, raw());
 }
 
-std::string DerivedPath::Opaque::to_string(const Store & store) const
+std::string DerivedPath::Opaque::to_string(const StoreDirConfig & store) const
 {
     return store.printStorePath(path);
 }
 
-std::string SingleDerivedPath::Built::to_string(const Store & store) const
+std::string SingleDerivedPath::Built::to_string(const StoreDirConfig & store) const
 {
     return drvPath->to_string(store) + "^" + output;
 }
 
-std::string SingleDerivedPath::Built::to_string_legacy(const Store & store) const
+std::string SingleDerivedPath::Built::to_string_legacy(const StoreDirConfig & store) const
 {
     return drvPath->to_string(store) + "!" + output;
 }
 
-std::string DerivedPath::Built::to_string(const Store & store) const
+std::string DerivedPath::Built::to_string(const StoreDirConfig & store) const
 {
     return drvPath->to_string(store)
         + '^'
         + outputs.to_string();
 }
 
-std::string DerivedPath::Built::to_string_legacy(const Store & store) const
+std::string DerivedPath::Built::to_string_legacy(const StoreDirConfig & store) const
 {
     return drvPath->to_string_legacy(store)
         + "!"
         + outputs.to_string();
 }
 
-std::string SingleDerivedPath::to_string(const Store & store) const
+std::string SingleDerivedPath::to_string(const StoreDirConfig & store) const
 {
     return std::visit(
         [&](const auto & req) { return req.to_string(store); },
         raw());
 }
 
-std::string DerivedPath::to_string(const Store & store) const
+std::string DerivedPath::to_string(const StoreDirConfig & store) const
 {
     return std::visit(
         [&](const auto & req) { return req.to_string(store); },
         raw());
 }
 
-std::string SingleDerivedPath::to_string_legacy(const Store & store) const
+std::string SingleDerivedPath::to_string_legacy(const StoreDirConfig & store) const
 {
     return std::visit(overloaded {
         [&](const SingleDerivedPath::Built & req) { return req.to_string_legacy(store); },
@@ -137,7 +138,7 @@ std::string SingleDerivedPath::to_string_legacy(const Store & store) const
     }, this->raw());
 }
 
-std::string DerivedPath::to_string_legacy(const Store & store) const
+std::string DerivedPath::to_string_legacy(const StoreDirConfig & store) const
 {
     return std::visit(overloaded {
         [&](const DerivedPath::Built & req) { return req.to_string_legacy(store); },
@@ -146,7 +147,7 @@ std::string DerivedPath::to_string_legacy(const Store & store) const
 }
 
 
-DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_view s)
+DerivedPath::Opaque DerivedPath::Opaque::parse(const StoreDirConfig & store, std::string_view s)
 {
     return {store.parseStorePath(s)};
 }
@@ -166,7 +167,7 @@ void drvRequireExperiment(
 }
 
 SingleDerivedPath::Built SingleDerivedPath::Built::parse(
-    const Store & store, ref<SingleDerivedPath> drv,
+    const StoreDirConfig & store, ref<SingleDerivedPath> drv,
     OutputNameView output,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -178,7 +179,7 @@ SingleDerivedPath::Built SingleDerivedPath::Built::parse(
 }
 
 DerivedPath::Built DerivedPath::Built::parse(
-    const Store & store, ref<SingleDerivedPath> drv,
+    const StoreDirConfig & store, ref<SingleDerivedPath> drv,
     OutputNameView outputsS,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -190,7 +191,7 @@ DerivedPath::Built DerivedPath::Built::parse(
 }
 
 static SingleDerivedPath parseWithSingle(
-    const Store & store, std::string_view s, std::string_view separator,
+    const StoreDirConfig & store, std::string_view s, std::string_view separator,
     const ExperimentalFeatureSettings & xpSettings)
 {
     size_t n = s.rfind(separator);
@@ -207,7 +208,7 @@ static SingleDerivedPath parseWithSingle(
 }
 
 SingleDerivedPath SingleDerivedPath::parse(
-    const Store & store,
+    const StoreDirConfig & store,
     std::string_view s,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -215,7 +216,7 @@ SingleDerivedPath SingleDerivedPath::parse(
 }
 
 SingleDerivedPath SingleDerivedPath::parseLegacy(
-    const Store & store,
+    const StoreDirConfig & store,
     std::string_view s,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -223,7 +224,7 @@ SingleDerivedPath SingleDerivedPath::parseLegacy(
 }
 
 static DerivedPath parseWith(
-    const Store & store, std::string_view s, std::string_view separator,
+    const StoreDirConfig & store, std::string_view s, std::string_view separator,
     const ExperimentalFeatureSettings & xpSettings)
 {
     size_t n = s.rfind(separator);
@@ -240,7 +241,7 @@ static DerivedPath parseWith(
 }
 
 DerivedPath DerivedPath::parse(
-    const Store & store,
+    const StoreDirConfig & store,
     std::string_view s,
     const ExperimentalFeatureSettings & xpSettings)
 {
@@ -248,7 +249,7 @@ DerivedPath DerivedPath::parse(
 }
 
 DerivedPath DerivedPath::parseLegacy(
-    const Store & store,
+    const StoreDirConfig & store,
     std::string_view s,
     const ExperimentalFeatureSettings & xpSettings)
 {
diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh
index 4d7033df2..b12f9734a 100644
--- a/src/libstore/derived-path.hh
+++ b/src/libstore/derived-path.hh
@@ -12,6 +12,9 @@
 
 namespace nix {
 
+struct StoreDirConfig;
+
+// TODO stop needing this, `toJSON` below should be pure
 class Store;
 
 /**
@@ -24,9 +27,9 @@ class Store;
 struct DerivedPathOpaque {
     StorePath path;
 
-    std::string to_string(const Store & store) const;
-    static DerivedPathOpaque parse(const Store & store, std::string_view);
-    nlohmann::json toJSON(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
+    static DerivedPathOpaque parse(const StoreDirConfig & store, std::string_view);
+    nlohmann::json toJSON(const StoreDirConfig & store) const;
 
     GENERATE_CMP(DerivedPathOpaque, me->path);
 };
@@ -59,18 +62,18 @@ struct SingleDerivedPathBuilt {
     /**
      * Uses `^` as the separator
      */
-    std::string to_string(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
     /**
      * Uses `!` as the separator
      */
-    std::string to_string_legacy(const Store & store) const;
+    std::string to_string_legacy(const StoreDirConfig & store) const;
     /**
      * The caller splits on the separator, so it works for both variants.
      *
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static SingleDerivedPathBuilt parse(
-        const Store & store, ref<SingleDerivedPath> drvPath,
+        const StoreDirConfig & store, ref<SingleDerivedPath> drvPath,
         OutputNameView outputs,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     nlohmann::json toJSON(Store & store) const;
@@ -120,18 +123,18 @@ struct SingleDerivedPath : _SingleDerivedPathRaw {
     /**
      * Uses `^` as the separator
      */
-    std::string to_string(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
     /**
      * Uses `!` as the separator
      */
-    std::string to_string_legacy(const Store & store) const;
+    std::string to_string_legacy(const StoreDirConfig & store) const;
     /**
      * Uses `^` as the separator
      *
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static SingleDerivedPath parse(
-        const Store & store,
+        const StoreDirConfig & store,
         std::string_view,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     /**
@@ -140,7 +143,7 @@ struct SingleDerivedPath : _SingleDerivedPathRaw {
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static SingleDerivedPath parseLegacy(
-        const Store & store,
+        const StoreDirConfig & store,
         std::string_view,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     nlohmann::json toJSON(Store & store) const;
@@ -182,18 +185,18 @@ struct DerivedPathBuilt {
     /**
      * Uses `^` as the separator
      */
-    std::string to_string(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
     /**
      * Uses `!` as the separator
      */
-    std::string to_string_legacy(const Store & store) const;
+    std::string to_string_legacy(const StoreDirConfig & store) const;
     /**
      * The caller splits on the separator, so it works for both variants.
      *
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static DerivedPathBuilt parse(
-        const Store & store, ref<SingleDerivedPath>,
+        const StoreDirConfig & store, ref<SingleDerivedPath>,
         std::string_view,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     nlohmann::json toJSON(Store & store) const;
@@ -242,18 +245,18 @@ struct DerivedPath : _DerivedPathRaw {
     /**
      * Uses `^` as the separator
      */
-    std::string to_string(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
     /**
      * Uses `!` as the separator
      */
-    std::string to_string_legacy(const Store & store) const;
+    std::string to_string_legacy(const StoreDirConfig & store) const;
     /**
      * Uses `^` as the separator
      *
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static DerivedPath parse(
-        const Store & store,
+        const StoreDirConfig & store,
         std::string_view,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
     /**
@@ -262,7 +265,7 @@ struct DerivedPath : _DerivedPathRaw {
      * @param xpSettings Stop-gap to avoid globals during unit tests.
      */
     static DerivedPath parseLegacy(
-        const Store & store,
+        const StoreDirConfig & store,
         std::string_view,
         const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 
diff --git a/src/libstore/length-prefixed-protocol-helper.hh b/src/libstore/length-prefixed-protocol-helper.hh
index 4061b0cd6..0cf950a47 100644
--- a/src/libstore/length-prefixed-protocol-helper.hh
+++ b/src/libstore/length-prefixed-protocol-helper.hh
@@ -10,7 +10,7 @@
 
 namespace nix {
 
-class Store;
+struct StoreDirConfig;
 
 /**
  * Reusable serialisers for serialization container types in a
@@ -44,8 +44,8 @@ struct LengthPrefixedProtoHelper;
 #define LENGTH_PREFIXED_PROTO_HELPER(Inner, T) \
     struct LengthPrefixedProtoHelper< Inner, T > \
     { \
-        static T read(const Store & store, typename Inner::ReadConn conn); \
-        static void write(const Store & store, typename Inner::WriteConn conn, const T & str); \
+        static T read(const StoreDirConfig & store, typename Inner::ReadConn conn); \
+        static void write(const StoreDirConfig & store, typename Inner::WriteConn conn, const T & str); \
     private: \
         template<typename U> using S = typename Inner::template Serialise<U>; \
     }
@@ -67,7 +67,7 @@ LENGTH_PREFIXED_PROTO_HELPER(Inner, _X);
 template<class Inner, typename T>
 std::vector<T>
 LengthPrefixedProtoHelper<Inner, std::vector<T>>::read(
-    const Store & store, typename Inner::ReadConn conn)
+    const StoreDirConfig & store, typename Inner::ReadConn conn)
 {
     std::vector<T> resSet;
     auto size = readNum<size_t>(conn.from);
@@ -80,7 +80,7 @@ LengthPrefixedProtoHelper<Inner, std::vector<T>>::read(
 template<class Inner, typename T>
 void
 LengthPrefixedProtoHelper<Inner, std::vector<T>>::write(
-    const Store & store, typename Inner::WriteConn conn, const std::vector<T> & resSet)
+    const StoreDirConfig & store, typename Inner::WriteConn conn, const std::vector<T> & resSet)
 {
     conn.to << resSet.size();
     for (auto & key : resSet) {
@@ -91,7 +91,7 @@ LengthPrefixedProtoHelper<Inner, std::vector<T>>::write(
 template<class Inner, typename T>
 std::set<T>
 LengthPrefixedProtoHelper<Inner, std::set<T>>::read(
-    const Store & store, typename Inner::ReadConn conn)
+    const StoreDirConfig & store, typename Inner::ReadConn conn)
 {
     std::set<T> resSet;
     auto size = readNum<size_t>(conn.from);
@@ -104,7 +104,7 @@ LengthPrefixedProtoHelper<Inner, std::set<T>>::read(
 template<class Inner, typename T>
 void
 LengthPrefixedProtoHelper<Inner, std::set<T>>::write(
-    const Store & store, typename Inner::WriteConn conn, const std::set<T> & resSet)
+    const StoreDirConfig & store, typename Inner::WriteConn conn, const std::set<T> & resSet)
 {
     conn.to << resSet.size();
     for (auto & key : resSet) {
@@ -115,7 +115,7 @@ LengthPrefixedProtoHelper<Inner, std::set<T>>::write(
 template<class Inner, typename K, typename V>
 std::map<K, V>
 LengthPrefixedProtoHelper<Inner, std::map<K, V>>::read(
-    const Store & store, typename Inner::ReadConn conn)
+    const StoreDirConfig & store, typename Inner::ReadConn conn)
 {
     std::map<K, V> resMap;
     auto size = readNum<size_t>(conn.from);
@@ -130,7 +130,7 @@ LengthPrefixedProtoHelper<Inner, std::map<K, V>>::read(
 template<class Inner, typename K, typename V>
 void
 LengthPrefixedProtoHelper<Inner, std::map<K, V>>::write(
-    const Store & store, typename Inner::WriteConn conn, const std::map<K, V> & resMap)
+    const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map<K, V> & resMap)
 {
     conn.to << resMap.size();
     for (auto & i : resMap) {
@@ -142,7 +142,7 @@ LengthPrefixedProtoHelper<Inner, std::map<K, V>>::write(
 template<class Inner, typename... Ts>
 std::tuple<Ts...>
 LengthPrefixedProtoHelper<Inner, std::tuple<Ts...>>::read(
-    const Store & store, typename Inner::ReadConn conn)
+    const StoreDirConfig & store, typename Inner::ReadConn conn)
 {
     return std::tuple<Ts...> {
         S<Ts>::read(store, conn)...,
@@ -152,7 +152,7 @@ LengthPrefixedProtoHelper<Inner, std::tuple<Ts...>>::read(
 template<class Inner, typename... Ts>
 void
 LengthPrefixedProtoHelper<Inner, std::tuple<Ts...>>::write(
-    const Store & store, typename Inner::WriteConn conn, const std::tuple<Ts...> & res)
+    const StoreDirConfig & store, typename Inner::WriteConn conn, const std::tuple<Ts...> & res)
 {
     std::apply([&]<typename... Us>(const Us &... args) {
         (S<Us>::write(store, conn, args), ...);
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index a5e9426f8..e091683dc 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -4,6 +4,7 @@
 #include "pathlocks.hh"
 #include "worker-protocol.hh"
 #include "derivations.hh"
+#include "realisation.hh"
 #include "nar-info.hh"
 #include "references.hh"
 #include "callback.hh"
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 1035691c7..9f63fbbb5 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -4,6 +4,7 @@
 #include "local-store.hh"
 #include "store-api.hh"
 #include "thread-pool.hh"
+#include "realisation.hh"
 #include "topo-sort.hh"
 #include "callback.hh"
 #include "closure.hh"
diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc
index af6837370..026e37647 100644
--- a/src/libstore/path-with-outputs.cc
+++ b/src/libstore/path-with-outputs.cc
@@ -5,7 +5,7 @@
 
 namespace nix {
 
-std::string StorePathWithOutputs::to_string(const Store & store) const
+std::string StorePathWithOutputs::to_string(const StoreDirConfig & store) const
 {
     return outputs.empty()
         ? store.printStorePath(path)
@@ -85,7 +85,7 @@ std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s)
 }
 
 
-StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs)
+StorePathWithOutputs parsePathWithOutputs(const StoreDirConfig & store, std::string_view pathWithOutputs)
 {
     auto [path, outputs] = parsePathWithOutputs(pathWithOutputs);
     return StorePathWithOutputs { store.parseStorePath(path), std::move(outputs) };
diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh
index 57e03252d..5f76a583a 100644
--- a/src/libstore/path-with-outputs.hh
+++ b/src/libstore/path-with-outputs.hh
@@ -6,6 +6,8 @@
 
 namespace nix {
 
+struct StoreDirConfig;
+
 /**
  * This is a deprecated old type just for use by the old CLI, and older
  * versions of the RPC protocols. In new code don't use it; you want
@@ -19,7 +21,7 @@ struct StorePathWithOutputs
     StorePath path;
     std::set<std::string> outputs;
 
-    std::string to_string(const Store & store) const;
+    std::string to_string(const StoreDirConfig & store) const;
 
     DerivedPath toDerivedPath() const;
 
@@ -32,14 +34,14 @@ std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs>)
 
 std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s);
 
-class Store;
-
 /**
  * Split a string specifying a derivation and a set of outputs
  * (/nix/store/hash-foo!out1,out2,...) into the derivation path
  * and the outputs.
  */
-StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
+StorePathWithOutputs parsePathWithOutputs(const StoreDirConfig & store, std::string_view pathWithOutputs);
+
+class Store;
 
 StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
 
diff --git a/src/libstore/serve-protocol-impl.hh b/src/libstore/serve-protocol-impl.hh
index a3ce81026..6f3b177ac 100644
--- a/src/libstore/serve-protocol-impl.hh
+++ b/src/libstore/serve-protocol-impl.hh
@@ -16,11 +16,11 @@ namespace nix {
 /* protocol-agnostic templates */
 
 #define SERVE_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \
-    TEMPLATE T ServeProto::Serialise< T >::read(const Store & store, ServeProto::ReadConn conn) \
+    TEMPLATE T ServeProto::Serialise< T >::read(const StoreDirConfig & store, ServeProto::ReadConn conn) \
     { \
         return LengthPrefixedProtoHelper<ServeProto, T >::read(store, conn); \
     } \
-    TEMPLATE void ServeProto::Serialise< T >::write(const Store & store, ServeProto::WriteConn conn, const T & t) \
+    TEMPLATE void ServeProto::Serialise< T >::write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t) \
     { \
         LengthPrefixedProtoHelper<ServeProto, T >::write(store, conn, t); \
     }
@@ -41,12 +41,12 @@ SERVE_USE_LENGTH_PREFIX_SERIALISER(
 template<typename T>
 struct ServeProto::Serialise
 {
-    static T read(const Store & store, ServeProto::ReadConn conn)
+    static T read(const StoreDirConfig & store, ServeProto::ReadConn conn)
     {
         return CommonProto::Serialise<T>::read(store,
             CommonProto::ReadConn { .from = conn.from });
     }
-    static void write(const Store & store, ServeProto::WriteConn conn, const T & t)
+    static void write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t)
     {
         CommonProto::Serialise<T>::write(store,
             CommonProto::WriteConn { .to = conn.to },
diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc
index 97a0ddf0e..e0ac80c4e 100644
--- a/src/libstore/serve-protocol.cc
+++ b/src/libstore/serve-protocol.cc
@@ -13,7 +13,7 @@ namespace nix {
 
 /* protocol-specific definitions */
 
-BuildResult ServeProto::Serialise<BuildResult>::read(const Store & store, ServeProto::ReadConn conn)
+BuildResult ServeProto::Serialise<BuildResult>::read(const StoreDirConfig & store, ServeProto::ReadConn conn)
 {
     BuildResult status;
     status.status = (BuildResult::Status) readInt(conn.from);
@@ -35,7 +35,7 @@ BuildResult ServeProto::Serialise<BuildResult>::read(const Store & store, ServeP
     return status;
 }
 
-void ServeProto::Serialise<BuildResult>::write(const Store & store, ServeProto::WriteConn conn, const BuildResult & status)
+void ServeProto::Serialise<BuildResult>::write(const StoreDirConfig & store, ServeProto::WriteConn conn, const BuildResult & status)
 {
     conn.to
         << status.status
diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh
index ba159f6e9..6e9d66e2d 100644
--- a/src/libstore/serve-protocol.hh
+++ b/src/libstore/serve-protocol.hh
@@ -13,7 +13,7 @@ namespace nix {
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
 
-class Store;
+struct StoreDirConfig;
 struct Source;
 
 // items being serialised
@@ -72,8 +72,8 @@ struct ServeProto
     // See `worker-protocol.hh` for a longer explanation.
 #if 0
     {
-        static T read(const Store & store, ReadConn conn);
-        static void write(const Store & store, WriteConn conn, const T & t);
+        static T read(const StoreDirConfig & store, ReadConn conn);
+        static void write(const StoreDirConfig & store, WriteConn conn, const T & t);
     };
 #endif
 
@@ -82,7 +82,7 @@ struct ServeProto
      * infer the type instead of having to write it down explicitly.
      */
     template<typename T>
-    static void write(const Store & store, WriteConn conn, const T & t)
+    static void write(const StoreDirConfig & store, WriteConn conn, const T & t)
     {
         ServeProto::Serialise<T>::write(store, conn, t);
     }
@@ -135,8 +135,8 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op)
 #define DECLARE_SERVE_SERIALISER(T) \
     struct ServeProto::Serialise< T > \
     { \
-        static T read(const Store & store, ServeProto::ReadConn conn); \
-        static void write(const Store & store, ServeProto::WriteConn conn, const T & t); \
+        static T read(const StoreDirConfig & store, ServeProto::ReadConn conn); \
+        static void write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t); \
     };
 
 template<>
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index a681bb6cf..e44376fa2 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -1,6 +1,8 @@
 #include "crypto.hh"
 #include "source-accessor.hh"
 #include "globals.hh"
+#include "derived-path.hh"
+#include "realisation.hh"
 #include "derivations.hh"
 #include "store-api.hh"
 #include "util.hh"
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index bee5ec16c..4342445ba 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -1,8 +1,6 @@
 #pragma once
 ///@file
 
-#include "nar-info.hh"
-#include "realisation.hh"
 #include "path.hh"
 #include "derived-path.hh"
 #include "hash.hh"
@@ -68,8 +66,13 @@ MakeError(SubstituterDisabled, Error);
 
 MakeError(InvalidStoreURI, Error);
 
+struct Realisation;
+struct RealisedPath;
+struct DrvOutput;
+
 struct BasicDerivation;
 struct Derivation;
+
 struct SourceAccessor;
 class NarInfoDiskCache;
 class Store;
@@ -811,7 +814,7 @@ void copyStorePath(
  */
 std::map<StorePath, StorePath> copyPaths(
     Store & srcStore, Store & dstStore,
-    const RealisedPath::Set &,
+    const std::set<RealisedPath> &,
     RepairFlag repair = NoRepair,
     CheckSigsFlag checkSigs = CheckSigs,
     SubstituteFlag substitute = NoSubstitute);
@@ -828,7 +831,7 @@ std::map<StorePath, StorePath> copyPaths(
  */
 void copyClosure(
     Store & srcStore, Store & dstStore,
-    const RealisedPath::Set & paths,
+    const std::set<RealisedPath> & paths,
     RepairFlag repair = NoRepair,
     CheckSigsFlag checkSigs = CheckSigs,
     SubstituteFlag substitute = NoSubstitute);
diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh
index c043588d6..026cc37bc 100644
--- a/src/libstore/worker-protocol-impl.hh
+++ b/src/libstore/worker-protocol-impl.hh
@@ -16,11 +16,11 @@ namespace nix {
 /* protocol-agnostic templates */
 
 #define WORKER_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \
-    TEMPLATE T WorkerProto::Serialise< T >::read(const Store & store, WorkerProto::ReadConn conn) \
+    TEMPLATE T WorkerProto::Serialise< T >::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) \
     { \
         return LengthPrefixedProtoHelper<WorkerProto, T >::read(store, conn); \
     } \
-    TEMPLATE void WorkerProto::Serialise< T >::write(const Store & store, WorkerProto::WriteConn conn, const T & t) \
+    TEMPLATE void WorkerProto::Serialise< T >::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t) \
     { \
         LengthPrefixedProtoHelper<WorkerProto, T >::write(store, conn, t); \
     }
@@ -41,12 +41,12 @@ WORKER_USE_LENGTH_PREFIX_SERIALISER(
 template<typename T>
 struct WorkerProto::Serialise
 {
-    static T read(const Store & store, WorkerProto::ReadConn conn)
+    static T read(const StoreDirConfig & store, WorkerProto::ReadConn conn)
     {
         return CommonProto::Serialise<T>::read(store,
             CommonProto::ReadConn { .from = conn.from });
     }
-    static void write(const Store & store, WorkerProto::WriteConn conn, const T & t)
+    static void write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t)
     {
         CommonProto::Serialise<T>::write(store,
             CommonProto::WriteConn { .to = conn.to },
diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc
index d618b9bd8..4edab7894 100644
--- a/src/libstore/worker-protocol.cc
+++ b/src/libstore/worker-protocol.cc
@@ -14,7 +14,7 @@ namespace nix {
 
 /* protocol-specific definitions */
 
-std::optional<TrustedFlag> WorkerProto::Serialise<std::optional<TrustedFlag>>::read(const Store & store, WorkerProto::ReadConn conn)
+std::optional<TrustedFlag> WorkerProto::Serialise<std::optional<TrustedFlag>>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn)
 {
     auto temp = readNum<uint8_t>(conn.from);
     switch (temp) {
@@ -29,7 +29,7 @@ std::optional<TrustedFlag> WorkerProto::Serialise<std::optional<TrustedFlag>>::r
     }
 }
 
-void WorkerProto::Serialise<std::optional<TrustedFlag>>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional<TrustedFlag> & optTrusted)
+void WorkerProto::Serialise<std::optional<TrustedFlag>>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const std::optional<TrustedFlag> & optTrusted)
 {
     if (!optTrusted)
         conn.to << (uint8_t)0;
@@ -48,7 +48,7 @@ void WorkerProto::Serialise<std::optional<TrustedFlag>>::write(const Store & sto
 }
 
 
-DerivedPath WorkerProto::Serialise<DerivedPath>::read(const Store & store, WorkerProto::ReadConn conn)
+DerivedPath WorkerProto::Serialise<DerivedPath>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn)
 {
     auto s = readString(conn.from);
     if (GET_PROTOCOL_MINOR(conn.version) >= 30) {
@@ -58,7 +58,7 @@ DerivedPath WorkerProto::Serialise<DerivedPath>::read(const Store & store, Worke
     }
 }
 
-void WorkerProto::Serialise<DerivedPath>::write(const Store & store, WorkerProto::WriteConn conn, const DerivedPath & req)
+void WorkerProto::Serialise<DerivedPath>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const DerivedPath & req)
 {
     if (GET_PROTOCOL_MINOR(conn.version) >= 30) {
         conn.to << req.to_string_legacy(store);
@@ -82,7 +82,7 @@ void WorkerProto::Serialise<DerivedPath>::write(const Store & store, WorkerProto
 }
 
 
-KeyedBuildResult WorkerProto::Serialise<KeyedBuildResult>::read(const Store & store, WorkerProto::ReadConn conn)
+KeyedBuildResult WorkerProto::Serialise<KeyedBuildResult>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn)
 {
     auto path = WorkerProto::Serialise<DerivedPath>::read(store, conn);
     auto br = WorkerProto::Serialise<BuildResult>::read(store, conn);
@@ -92,14 +92,14 @@ KeyedBuildResult WorkerProto::Serialise<KeyedBuildResult>::read(const Store & st
     };
 }
 
-void WorkerProto::Serialise<KeyedBuildResult>::write(const Store & store, WorkerProto::WriteConn conn, const KeyedBuildResult & res)
+void WorkerProto::Serialise<KeyedBuildResult>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const KeyedBuildResult & res)
 {
     WorkerProto::write(store, conn, res.path);
     WorkerProto::write(store, conn, static_cast<const BuildResult &>(res));
 }
 
 
-BuildResult WorkerProto::Serialise<BuildResult>::read(const Store & store, WorkerProto::ReadConn conn)
+BuildResult WorkerProto::Serialise<BuildResult>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn)
 {
     BuildResult res;
     res.status = (BuildResult::Status) readInt(conn.from);
@@ -121,7 +121,7 @@ BuildResult WorkerProto::Serialise<BuildResult>::read(const Store & store, Worke
     return res;
 }
 
-void WorkerProto::Serialise<BuildResult>::write(const Store & store, WorkerProto::WriteConn conn, const BuildResult & res)
+void WorkerProto::Serialise<BuildResult>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const BuildResult & res)
 {
     conn.to
         << res.status
@@ -142,7 +142,7 @@ void WorkerProto::Serialise<BuildResult>::write(const Store & store, WorkerProto
 }
 
 
-ValidPathInfo WorkerProto::Serialise<ValidPathInfo>::read(const Store & store, ReadConn conn)
+ValidPathInfo WorkerProto::Serialise<ValidPathInfo>::read(const StoreDirConfig & store, ReadConn conn)
 {
     auto path = WorkerProto::Serialise<StorePath>::read(store, conn);
     return ValidPathInfo {
@@ -151,14 +151,14 @@ ValidPathInfo WorkerProto::Serialise<ValidPathInfo>::read(const Store & store, R
     };
 }
 
-void WorkerProto::Serialise<ValidPathInfo>::write(const Store & store, WriteConn conn, const ValidPathInfo & pathInfo)
+void WorkerProto::Serialise<ValidPathInfo>::write(const StoreDirConfig & store, WriteConn conn, const ValidPathInfo & pathInfo)
 {
     WorkerProto::write(store, conn, pathInfo.path);
     WorkerProto::write(store, conn, static_cast<const UnkeyedValidPathInfo &>(pathInfo));
 }
 
 
-UnkeyedValidPathInfo WorkerProto::Serialise<UnkeyedValidPathInfo>::read(const Store & store, ReadConn conn)
+UnkeyedValidPathInfo WorkerProto::Serialise<UnkeyedValidPathInfo>::read(const StoreDirConfig & store, ReadConn conn)
 {
     auto deriver = readString(conn.from);
     auto narHash = Hash::parseAny(readString(conn.from), htSHA256);
@@ -174,7 +174,7 @@ UnkeyedValidPathInfo WorkerProto::Serialise<UnkeyedValidPathInfo>::read(const St
     return info;
 }
 
-void WorkerProto::Serialise<UnkeyedValidPathInfo>::write(const Store & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo)
+void WorkerProto::Serialise<UnkeyedValidPathInfo>::write(const StoreDirConfig & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo)
 {
     conn.to
         << (pathInfo.deriver ? store.printStorePath(*pathInfo.deriver) : "")
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index dcd54ad16..9b02aa2b5 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -24,7 +24,7 @@ namespace nix {
 #define STDERR_RESULT         0x52534c54
 
 
-class Store;
+struct StoreDirConfig;
 struct Source;
 
 // items being serialised
@@ -100,8 +100,8 @@ struct WorkerProto
     // This makes for a quicker debug cycle, as desired.
 #if 0
     {
-        static T read(const Store & store, ReadConn conn);
-        static void write(const Store & store, WriteConn conn, const T & t);
+        static T read(const StoreDirConfig & store, ReadConn conn);
+        static void write(const StoreDirConfig & store, WriteConn conn, const T & t);
     };
 #endif
 
@@ -110,7 +110,7 @@ struct WorkerProto
      * infer the type instead of having to write it down explicitly.
      */
     template<typename T>
-    static void write(const Store & store, WriteConn conn, const T & t)
+    static void write(const StoreDirConfig & store, WriteConn conn, const T & t)
     {
         WorkerProto::Serialise<T>::write(store, conn, t);
     }
@@ -197,8 +197,8 @@ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op)
 #define DECLARE_WORKER_SERIALISER(T) \
     struct WorkerProto::Serialise< T > \
     { \
-        static T read(const Store & store, WorkerProto::ReadConn conn); \
-        static void write(const Store & store, WorkerProto::WriteConn conn, const T & t); \
+        static T read(const StoreDirConfig & store, WorkerProto::ReadConn conn); \
+        static void write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t); \
     };
 
 template<>
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 60bc08146..c46095a14 100644
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -13,6 +13,7 @@
 #include "store-api.hh"
 #include "local-fs-store.hh"
 #include "globals.hh"
+#include "realisation.hh"
 #include "derivations.hh"
 #include "util.hh"
 #include "shared.hh"
diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
index 7f2bb93b6..b64af758f 100644
--- a/src/nix-copy-closure/nix-copy-closure.cc
+++ b/src/nix-copy-closure/nix-copy-closure.cc
@@ -1,4 +1,5 @@
 #include "shared.hh"
+#include "realisation.hh"
 #include "store-api.hh"
 #include "legacy.hh"
 

From 257b768436a0e8ab7887f9b790c5b92a7fe51ef5 Mon Sep 17 00:00:00 2001
From: Felix Uhl <felix.uhl@outlook.com>
Date: Sun, 9 Jul 2023 22:16:21 +0200
Subject: [PATCH 014/421] Enable using human-readable name in nix profile

---
 doc/manual/src/release-notes/rl-next.md |   2 +
 src/libutil/tests/url-name.cc           |  64 ++++++++++
 src/libutil/url-name.cc                 |  46 +++++++
 src/libutil/url-name.hh                 |  20 +++
 src/nix/profile-list.md                 |  10 +-
 src/nix/profile-remove.md               |  15 +--
 src/nix/profile-upgrade.md              |  10 +-
 src/nix/profile.cc                      | 160 ++++++++++++++++--------
 tests/functional/nix-profile.sh         |  21 ++--
 9 files changed, 274 insertions(+), 74 deletions(-)
 create mode 100644 src/libutil/tests/url-name.cc
 create mode 100644 src/libutil/url-name.cc
 create mode 100644 src/libutil/url-name.hh

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index 85e180e37..d50da32cd 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -64,3 +64,5 @@
     ```
 
   This makes it match `nix derivation show`, which also maps store paths to information.
+
+- [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) now uses names to refer to installed packages when running [`list`](@docroot@/command-ref/new-cli/nix3-profile-list.md), [`remove`](@docroot@/command-ref/new-cli/nix3-profile-remove.md) or [`upgrade`](@docroot@/command-ref/new-cli/nix3-profile-upgrade.md) as opposed to indices. Indices are deprecated and will be removed in a future version.
diff --git a/src/libutil/tests/url-name.cc b/src/libutil/tests/url-name.cc
new file mode 100644
index 000000000..6ee66e826
--- /dev/null
+++ b/src/libutil/tests/url-name.cc
@@ -0,0 +1,64 @@
+#include "url-name.hh"
+#include <gtest/gtest.h>
+
+namespace nix {
+
+/* ----------- tests for url-name.hh --------------------------------------------------*/
+
+    TEST(getNameFromURL, getsNameFromURL) {
+        ASSERT_EQ(getNameFromURL(parseURL("path:/home/user/project")), "project");
+        ASSERT_EQ(getNameFromURL(parseURL("path:~/repos/nixpkgs#packages.x86_64-linux.hello")), "hello");
+        ASSERT_EQ(getNameFromURL(parseURL("path:.#nonStandardAttr.mylaptop")), "nonStandardAttr.mylaptop");
+        ASSERT_EQ(getNameFromURL(parseURL("path:./repos/myflake#nonStandardAttr.mylaptop")), "nonStandardAttr.mylaptop");
+
+        ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nixpkgs#packages.x86_64-linux.hello")), "hello");
+        ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nixpkgs#hello")), "hello");
+        ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nix#packages.x86_64-linux.default")), "nix");
+        ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nix#")), "nix");
+        ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nix")), "nix");
+        ASSERT_EQ(getNameFromURL(parseURL("github:cachix/devenv/main#packages.x86_64-linux.default")), "devenv");
+        ASSERT_EQ(getNameFromURL(parseURL("github:edolstra/nix-warez?rev=1234&dir=blender&ref=master")), "blender");
+
+        ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nixpkgs#packages.x86_64-linux.hello")), "hello");
+        ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nixpkgs#hello")), "hello");
+        ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nix#packages.x86_64-linux.default")), "nix");
+        ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nix#")), "nix");
+        ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nix")), "nix");
+        ASSERT_EQ(getNameFromURL(parseURL("gitlab:cachix/devenv/main#packages.x86_64-linux.default")), "devenv");
+
+        ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nixpkgs#packages.x86_64-linux.hello")), "hello");
+        ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nixpkgs#hello")), "hello");
+        ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nix#packages.x86_64-linux.default")), "nix");
+        ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nix#")), "nix");
+        ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nix")), "nix");
+        ASSERT_EQ(getNameFromURL(parseURL("sourcehut:cachix/devenv/main#packages.x86_64-linux.default")), "devenv");
+
+        ASSERT_EQ(getNameFromURL(parseURL("git://github.com/edolstra/dwarffs")), "dwarffs");
+        ASSERT_EQ(getNameFromURL(parseURL("git://github.com/edolstra/nix-warez?dir=blender")), "blender");
+        ASSERT_EQ(getNameFromURL(parseURL("git+file:///home/user/project")), "project");
+        ASSERT_EQ(getNameFromURL(parseURL("git+file:///home/user/project?ref=fa1e2d23a22")), "project");
+        ASSERT_EQ(getNameFromURL(parseURL("git+ssh://git@github.com/someuser/my-repo#")), "my-repo");
+        ASSERT_EQ(getNameFromURL(parseURL("git+git://github.com/someuser/my-repo?rev=v1.2.3")), "my-repo");
+        ASSERT_EQ(getNameFromURL(parseURL("git+ssh:///home/user/project?dir=subproject&rev=v2.4")), "subproject");
+        ASSERT_EQ(getNameFromURL(parseURL("git+http://not-even-real#packages.x86_64-linux.hello")), "hello");
+        ASSERT_EQ(getNameFromURL(parseURL("git+https://not-even-real#packages.aarch64-darwin.hello")), "hello");
+
+        ASSERT_EQ(getNameFromURL(parseURL("tarball+http://github.com/NixOS/nix/archive/refs/tags/2.18.1#packages.x86_64-linux.jq")), "jq");
+        ASSERT_EQ(getNameFromURL(parseURL("tarball+https://github.com/NixOS/nix/archive/refs/tags/2.18.1#packages.x86_64-linux.hg")), "hg");
+        ASSERT_EQ(getNameFromURL(parseURL("tarball+file:///home/user/Downloads/nixpkgs-2.18.1#packages.aarch64-darwin.ripgrep")), "ripgrep");
+
+        ASSERT_EQ(getNameFromURL(parseURL("https://github.com/NixOS/nix/archive/refs/tags/2.18.1.tar.gz#packages.x86_64-linux.pv")), "pv");
+        ASSERT_EQ(getNameFromURL(parseURL("http://github.com/NixOS/nix/archive/refs/tags/2.18.1.tar.gz#packages.x86_64-linux.pv")), "pv");
+
+        ASSERT_EQ(getNameFromURL(parseURL("file:///home/user/project?ref=fa1e2d23a22")), "project");
+        ASSERT_EQ(getNameFromURL(parseURL("file+file:///home/user/project?ref=fa1e2d23a22")), "project");
+        ASSERT_EQ(getNameFromURL(parseURL("file+http://not-even-real#packages.x86_64-linux.hello")), "hello");
+        ASSERT_EQ(getNameFromURL(parseURL("file+http://gitfantasy.com/org/user/notaflake")), "notaflake");
+        ASSERT_EQ(getNameFromURL(parseURL("file+https://not-even-real#packages.aarch64-darwin.hello")), "hello");
+
+        ASSERT_EQ(getNameFromURL(parseURL("https://www.github.com/")), std::nullopt);
+        ASSERT_EQ(getNameFromURL(parseURL("path:.")), std::nullopt);
+        ASSERT_EQ(getNameFromURL(parseURL("file:.#")), std::nullopt);
+        ASSERT_EQ(getNameFromURL(parseURL("path:.#packages.x86_64-linux.default")), std::nullopt);
+    }
+}
diff --git a/src/libutil/url-name.cc b/src/libutil/url-name.cc
new file mode 100644
index 000000000..ab65e78df
--- /dev/null
+++ b/src/libutil/url-name.cc
@@ -0,0 +1,46 @@
+#include "url-name.hh"
+#include <regex>
+#include <iostream>
+
+namespace nix {
+
+static const std::string attributeNamePattern("[a-z0-9_-]+");
+static const std::regex lastAttributeRegex("(?:" + attributeNamePattern + "\\.)*(?!default)(" + attributeNamePattern +")");
+static const std::string pathSegmentPattern("[a-zA-Z0-9_-]+");
+static const std::regex lastPathSegmentRegex(".*/(" + pathSegmentPattern +")");
+static const std::regex secondPathSegmentRegex("(?:" + pathSegmentPattern + ")/(" + pathSegmentPattern +")(?:/.*)?");
+static const std::regex gitProviderRegex("github|gitlab|sourcehut");
+static const std::regex gitSchemeRegex("git($|\\+.*)");
+
+std::optional<std::string> getNameFromURL(ParsedURL url) {
+  std::smatch match;
+
+  /* If there is a dir= argument, use its value */
+  if (url.query.count("dir") > 0)
+    return url.query.at("dir");
+
+  /* If the fragment isn't a "default" and contains two attribute elements, use the last one */
+  if (std::regex_match(url.fragment, match, lastAttributeRegex))
+    return match.str(1);
+
+  /* If this is a github/gitlab/sourcehut flake, use the repo name */
+  if (std::regex_match(url.scheme, gitProviderRegex) && std::regex_match(url.path, match, secondPathSegmentRegex))
+    return match.str(1);
+
+  /* If it is a regular git flake, use the directory name */
+  if (std::regex_match(url.scheme, gitSchemeRegex) && std::regex_match(url.path, match, lastPathSegmentRegex))
+    return match.str(1);
+
+  /* If everything failed but there is a non-default fragment, use it in full */
+  if (!url.fragment.empty() && !hasSuffix(url.fragment, "default"))
+    return url.fragment;
+
+  /* If there is no fragment, take the last element of the path */
+  if (std::regex_match(url.path, match, lastPathSegmentRegex))
+    return match.str(1);
+
+  /* If even that didn't work, the URL does not contain enough info to determine a useful name */
+  return {};
+}
+
+}
diff --git a/src/libutil/url-name.hh b/src/libutil/url-name.hh
new file mode 100644
index 000000000..188b951e5
--- /dev/null
+++ b/src/libutil/url-name.hh
@@ -0,0 +1,20 @@
+#include "url.hh"
+#include "url-parts.hh"
+#include "util.hh"
+#include "split.hh"
+
+namespace nix {
+
+/**
+ * Try to extract a reasonably unique and meaningful, human-readable
+ * name of a flake output from a parsed URL.
+ * When nullopt is returned, the callsite should use information available
+ * to it outside of the URL to determine a useful name.
+ * This is a heuristic approach intended for user interfaces.
+ * @return nullopt if the extracted name is not useful to identify a
+ * flake output, for example because it is empty or "default".
+ * Otherwise returns the extracted name.
+ */
+std::optional<std::string> getNameFromURL(ParsedURL url);
+
+}
diff --git a/src/nix/profile-list.md b/src/nix/profile-list.md
index 5d7fcc0ec..facfdf0d6 100644
--- a/src/nix/profile-list.md
+++ b/src/nix/profile-list.md
@@ -6,12 +6,14 @@ R""(
 
   ```console
   # nix profile list
+  Name:               gdb
   Index:              0
   Flake attribute:    legacyPackages.x86_64-linux.gdb
   Original flake URL: flake:nixpkgs
   Locked flake URL:   github:NixOS/nixpkgs/7b38b03d76ab71bdc8dc325e3f6338d984cc35ca
   Store paths:        /nix/store/indzcw5wvlhx6vwk7k4iq29q15chvr3d-gdb-11.1
 
+  Name:               blender-bin
   Index:              1
   Flake attribute:    packages.x86_64-linux.default
   Original flake URL: flake:blender-bin
@@ -26,7 +28,7 @@ R""(
   # nix build github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender#packages.x86_64-linux.default
   ```
 
-  will build the package with index 1 shown above.
+  will build the package `blender-bin` shown above.
 
 # Description
 
@@ -34,10 +36,14 @@ This command shows what packages are currently installed in a
 profile. For each installed package, it shows the following
 information:
 
-* `Index`: An integer that can be used to unambiguously identify the
+* `Name`: A unique name used to unambiguously identify the
   package in invocations of `nix profile remove` and `nix profile
   upgrade`.
 
+* `Index`: An integer that can be used to unambiguously identify the
+  package in invocations of `nix profile remove` and `nix profile upgrade`.
+  (*Deprecated, will be removed in a future version in favor of `Name`.*)
+
 * `Flake attribute`: The flake output attribute path that provides the
   package (e.g. `packages.x86_64-linux.hello`).
 
diff --git a/src/nix/profile-remove.md b/src/nix/profile-remove.md
index ba85441d8..c994b79bd 100644
--- a/src/nix/profile-remove.md
+++ b/src/nix/profile-remove.md
@@ -2,18 +2,19 @@ R""(
 
 # Examples
 
-* Remove a package by position:
+* Remove a package by name:
+
+  ```console
+  # nix profile remove hello
+  ```
+
+* Remove a package by index
+  *(deprecated, will be removed in a future version)*:
 
   ```console
   # nix profile remove 3
   ```
 
-* Remove a package by attribute path:
-
-  ```console
-  # nix profile remove packages.x86_64-linux.hello
-  ```
-
 * Remove all packages:
 
   ```console
diff --git a/src/nix/profile-upgrade.md b/src/nix/profile-upgrade.md
index 39cca428b..47103edfc 100644
--- a/src/nix/profile-upgrade.md
+++ b/src/nix/profile-upgrade.md
@@ -9,18 +9,16 @@ R""(
   # nix profile upgrade '.*'
   ```
 
-* Upgrade a specific package:
+* Upgrade a specific package by name:
 
   ```console
-  # nix profile upgrade packages.x86_64-linux.hello
+  # nix profile upgrade hello
   ```
 
-* Upgrade a specific profile element by number:
+* Upgrade a specific package by index
+  *(deprecated, will be removed in a future version)*:
 
   ```console
-  # nix profile list
-  0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify …
-
   # nix profile upgrade 0
   ```
 
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 476ddcd60..48a481858 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -10,6 +10,8 @@
 #include "../nix-env/user-env.hh"
 #include "profiles.hh"
 #include "names.hh"
+#include "url.hh"
+#include "url-name.hh"
 
 #include <nlohmann/json.hpp>
 #include <regex>
@@ -43,6 +45,7 @@ const int defaultPriority = 5;
 struct ProfileElement
 {
     StorePathSet storePaths;
+    std::string name;
     std::optional<ProfileElementSource> source;
     bool active = true;
     int priority = defaultPriority;
@@ -116,6 +119,8 @@ struct ProfileManifest
 
         if (pathExists(manifestPath)) {
             auto json = nlohmann::json::parse(readFile(manifestPath));
+            /* Keep track of alreay found names to allow preventing duplicates */
+            std::set<std::string> foundNames;
 
             auto version = json.value("version", 0);
             std::string sUrl;
@@ -149,6 +154,25 @@ struct ProfileManifest
                         e["outputs"].get<ExtendedOutputsSpec>()
                     };
                 }
+
+                std::string nameCandidate = element.identifier();
+                if (e.contains("name")) {
+                    nameCandidate = e["name"];
+                }
+                else if (element.source) {
+                    auto url = parseURL(element.source->to_string());
+                    auto name = getNameFromURL(url);
+                    if (name)
+                        nameCandidate = *name;
+                }
+
+                auto finalName = nameCandidate;
+                for (int i = 1; foundNames.contains(finalName); ++i) {
+                    finalName = nameCandidate + std::to_string(i);
+                }
+                element.name = finalName;
+                foundNames.insert(element.name);
+
                 elements.emplace_back(std::move(element));
             }
         }
@@ -163,6 +187,7 @@ struct ProfileManifest
             for (auto & drvInfo : drvInfos) {
                 ProfileElement element;
                 element.storePaths = {drvInfo.queryOutPath()};
+                element.name = element.identifier();
                 elements.emplace_back(std::move(element));
             }
         }
@@ -451,15 +476,25 @@ public:
     {
         std::vector<Matcher> res;
 
+        auto anyIndexMatchers = false;
+
         for (auto & s : _matchers) {
-            if (auto n = string2Int<size_t>(s))
+            if (auto n = string2Int<size_t>(s)) {
                 res.push_back(*n);
+                anyIndexMatchers = true;
+            }
             else if (store->isStorePath(s))
                 res.push_back(s);
             else
                 res.push_back(RegexPattern{s,std::regex(s, std::regex::extended | std::regex::icase)});
         }
 
+        if (anyIndexMatchers) {
+            warn("Indices are deprecated and will be removed in a future version!\n"
+                 "         Refer to packages by their `Name` as printed by `nix profile list`.\n"
+                 "         See https://github.com/NixOS/nix/issues/9171 for more information.");
+        }
+
         return res;
     }
 
@@ -471,8 +506,7 @@ public:
             } else if (auto path = std::get_if<Path>(&matcher)) {
                 if (element.storePaths.count(store.parseStorePath(*path))) return true;
             } else if (auto regex = std::get_if<RegexPattern>(&matcher)) {
-                if (element.source
-                    && std::regex_match(element.source->attrPath, regex->reg))
+                if (std::regex_match(element.name, regex->reg))
                     return true;
             }
         }
@@ -556,62 +590,83 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
         Installables installables;
         std::vector<size_t> indices;
 
+        auto matchedCount = 0;
         auto upgradedCount = 0;
 
         for (size_t i = 0; i < manifest.elements.size(); ++i) {
             auto & element(manifest.elements[i]);
-            if (element.source
-                && !element.source->originalRef.input.isLocked()
-                && matches(*store, element, i, matchers))
-            {
-                upgradedCount++;
-
-                Activity act(*logger, lvlChatty, actUnknown,
-                    fmt("checking '%s' for updates", element.source->attrPath));
-
-                auto installable = make_ref<InstallableFlake>(
-                    this,
-                    getEvalState(),
-                    FlakeRef(element.source->originalRef),
-                    "",
-                    element.source->outputs,
-                    Strings{element.source->attrPath},
-                    Strings{},
-                    lockFlags);
-
-                auto derivedPaths = installable->toDerivedPaths();
-                if (derivedPaths.empty()) continue;
-                auto * infop = dynamic_cast<ExtraPathInfoFlake *>(&*derivedPaths[0].info);
-                // `InstallableFlake` should use `ExtraPathInfoFlake`.
-                assert(infop);
-                auto & info = *infop;
-
-                if (element.source->lockedRef == info.flake.lockedRef) continue;
-
-                printInfo("upgrading '%s' from flake '%s' to '%s'",
-                    element.source->attrPath, element.source->lockedRef, info.flake.lockedRef);
-
-                element.source = ProfileElementSource {
-                    .originalRef = installable->flakeRef,
-                    .lockedRef = info.flake.lockedRef,
-                    .attrPath = info.value.attrPath,
-                    .outputs = installable->extendedOutputsSpec,
-                };
-
-                installables.push_back(installable);
-                indices.push_back(i);
+            if (!matches(*store, element, i, matchers)) {
+                continue;
             }
+
+            matchedCount++;
+
+            if (!element.source) {
+                warn(
+                    "Found package '%s', but it was not installed from a flake, so it can't be checked for upgrades!",
+                    element.identifier()
+                );
+                continue;
+            }
+            if (element.source->originalRef.input.isLocked()) {
+                warn(
+                    "Found package '%s', but it was installed from a locked flake reference so it can't be upgraded!",
+                    element.identifier()
+                );
+                continue;
+            }
+
+            upgradedCount++;
+
+            Activity act(*logger, lvlChatty, actUnknown,
+                fmt("checking '%s' for updates", element.source->attrPath));
+
+            auto installable = make_ref<InstallableFlake>(
+                this,
+                getEvalState(),
+                FlakeRef(element.source->originalRef),
+                "",
+                element.source->outputs,
+                Strings{element.source->attrPath},
+                Strings{},
+                lockFlags);
+
+            auto derivedPaths = installable->toDerivedPaths();
+            if (derivedPaths.empty()) continue;
+            auto * infop = dynamic_cast<ExtraPathInfoFlake *>(&*derivedPaths[0].info);
+            // `InstallableFlake` should use `ExtraPathInfoFlake`.
+            assert(infop);
+            auto & info = *infop;
+
+            if (element.source->lockedRef == info.flake.lockedRef) continue;
+
+            printInfo("upgrading '%s' from flake '%s' to '%s'",
+                element.source->attrPath, element.source->lockedRef, info.flake.lockedRef);
+
+            element.source = ProfileElementSource {
+                .originalRef = installable->flakeRef,
+                .lockedRef = info.flake.lockedRef,
+                .attrPath = info.value.attrPath,
+                .outputs = installable->extendedOutputsSpec,
+            };
+
+            installables.push_back(installable);
+            indices.push_back(i);
         }
 
         if (upgradedCount == 0) {
-            for (auto & matcher : matchers) {
-                if (const size_t * index = std::get_if<size_t>(&matcher)){
-                    warn("'%d' is not a valid index", *index);
-                } else if (const Path * path = std::get_if<Path>(&matcher)){
-                    warn("'%s' does not match any paths", *path);
-                } else if (const RegexPattern * regex = std::get_if<RegexPattern>(&matcher)){
-                    warn("'%s' does not match any packages", regex->pattern);
+            if (matchedCount == 0) {
+                for (auto & matcher : matchers) {
+                    if (const size_t * index = std::get_if<size_t>(&matcher)){
+                        warn("'%d' is not a valid index", *index);
+                    } else if (const Path * path = std::get_if<Path>(&matcher)){
+                        warn("'%s' does not match any paths", *path);
+                    } else if (const RegexPattern * regex = std::get_if<RegexPattern>(&matcher)){
+                        warn("'%s' does not match any packages", regex->pattern);
+                    }
                 }
+            } else {
+                warn("Found some packages but none of them could be upgraded.");
             }
             warn ("Use 'nix profile list' to see the current profile.");
         }
@@ -657,9 +712,10 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro
             for (size_t i = 0; i < manifest.elements.size(); ++i) {
                 auto & element(manifest.elements[i]);
                 if (i) logger->cout("");
-                logger->cout("Index:              " ANSI_BOLD "%s" ANSI_NORMAL "%s",
-                    i,
+                logger->cout("Name:               " ANSI_BOLD "%s" ANSI_NORMAL "%s",
+                    element.name,
                     element.active ? "" : " " ANSI_RED "(inactive)" ANSI_NORMAL);
+                logger->cout("Index:              %s", i);
                 if (element.source) {
                     logger->cout("Flake attribute:    %s%s", element.source->attrPath, element.source->outputs.to_string());
                     logger->cout("Original flake URL: %s", element.source->originalRef.to_string());
diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh
index 7c478a0cd..1fdbfb644 100644
--- a/tests/functional/nix-profile.sh
+++ b/tests/functional/nix-profile.sh
@@ -47,7 +47,7 @@ cp ./config.nix $flake1Dir/
 
 # Test upgrading from nix-env.
 nix-env -f ./user-envs.nix -i foo-1.0
-nix profile list | grep -A2 'Index:.*0' | grep 'Store paths:.*foo-1.0'
+nix profile list | grep -A2 'Name:.*foo' | grep 'Store paths:.*foo-1.0'
 nix profile install $flake1Dir -L
 nix profile list | grep -A4 'Index:.*1' | grep 'Locked flake URL:.*narHash'
 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]]
@@ -81,7 +81,7 @@ nix profile rollback
 
 # Test uninstall.
 [ -e $TEST_HOME/.nix-profile/bin/foo ]
-nix profile remove 0
+nix profile remove foo
 (! [ -e $TEST_HOME/.nix-profile/bin/foo ])
 nix profile history | grep 'foo: 1.0 -> ∅'
 nix profile diff-closures | grep 'Version 3 -> 4'
@@ -93,6 +93,13 @@ nix profile remove 1
 nix profile install $(nix-build --no-out-link ./simple.nix)
 [[ $(cat $TEST_HOME/.nix-profile/hello) = "Hello World!" ]]
 
+# Test packages with same name from different sources
+mkdir $TEST_ROOT/simple-too
+cp ./simple.nix ./config.nix simple.builder.sh $TEST_ROOT/simple-too
+nix profile install --file $TEST_ROOT/simple-too/simple.nix ''
+nix profile list | grep -A4 'Name:.*simple' | grep 'Name:.*simple1'
+nix profile remove simple1
+
 # Test wipe-history.
 nix profile wipe-history
 [[ $(nix profile history | grep Version | wc -l) -eq 1 ]]
@@ -104,7 +111,7 @@ nix profile upgrade 0
 nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 3.0, 3.0-man"
 
 # Test new install of CA package.
-nix profile remove 0
+nix profile remove flake1
 printf 4.0 > $flake1Dir/version
 printf Utrecht > $flake1Dir/who
 nix profile install $flake1Dir
@@ -112,26 +119,26 @@ nix profile install $flake1Dir
 [[ $(nix path-info --json $(realpath $TEST_HOME/.nix-profile/bin/hello) | jq -r .[].ca) =~ fixed:r:sha256: ]]
 
 # Override the outputs.
-nix profile remove 0 1
+nix profile remove simple flake1
 nix profile install "$flake1Dir^*"
 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Utrecht" ]]
 [ -e $TEST_HOME/.nix-profile/share/man ]
 [ -e $TEST_HOME/.nix-profile/include ]
 
 printf Nix > $flake1Dir/who
-nix profile upgrade 0
+nix profile upgrade flake1
 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Nix" ]]
 [ -e $TEST_HOME/.nix-profile/share/man ]
 [ -e $TEST_HOME/.nix-profile/include ]
 
-nix profile remove 0
+nix profile remove flake1
 nix profile install "$flake1Dir^man"
 (! [ -e $TEST_HOME/.nix-profile/bin/hello ])
 [ -e $TEST_HOME/.nix-profile/share/man ]
 (! [ -e $TEST_HOME/.nix-profile/include ])
 
 # test priority
-nix profile remove 0
+nix profile remove flake1
 
 # Make another flake.
 flake2Dir=$TEST_ROOT/flake2

From 9c0a09f09fbb930483b26f60f8552fbe5236b777 Mon Sep 17 00:00:00 2001
From: Bob van der Linden <bobvanderlinden@gmail.com>
Date: Sun, 1 Oct 2023 22:09:55 +0200
Subject: [PATCH 015/421] allow ^ in URLs

Users may select specific outputs using the ^output syntax or selecting
any output using ^*.

URL parsing currently doesn't support these kinds of output references:
parsing will fail.

Currently `queryRegex` was reused for URL fragments, which didn't
include support for ^. Now queryRegex has been split from fragmentRegex,
where only the fragmentRegex supports ^.
---
 src/libexpr/flake/flakeref.cc   | 2 +-
 src/libutil/tests/url-name.cc   | 3 +++
 src/libutil/url-name.cc         | 5 +++--
 src/libutil/url-parts.hh        | 1 +
 src/libutil/url.cc              | 2 +-
 tests/functional/nix-profile.sh | 1 +
 6 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc
index 16f45ace7..49d6940b1 100644
--- a/src/libexpr/flake/flakeref.cc
+++ b/src/libexpr/flake/flakeref.cc
@@ -190,7 +190,7 @@ std::optional<std::pair<FlakeRef, std::string>> parseFlakeIdRef(
 
     static std::regex flakeRegex(
         "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)"
-        + "(?:#(" + queryRegex + "))?",
+        + "(?:#(" + fragmentRegex + "))?",
         std::regex::ECMAScript);
 
     if (std::regex_match(url, match, flakeRegex)) {
diff --git a/src/libutil/tests/url-name.cc b/src/libutil/tests/url-name.cc
index 6ee66e826..f637efa89 100644
--- a/src/libutil/tests/url-name.cc
+++ b/src/libutil/tests/url-name.cc
@@ -10,6 +10,8 @@ namespace nix {
         ASSERT_EQ(getNameFromURL(parseURL("path:~/repos/nixpkgs#packages.x86_64-linux.hello")), "hello");
         ASSERT_EQ(getNameFromURL(parseURL("path:.#nonStandardAttr.mylaptop")), "nonStandardAttr.mylaptop");
         ASSERT_EQ(getNameFromURL(parseURL("path:./repos/myflake#nonStandardAttr.mylaptop")), "nonStandardAttr.mylaptop");
+        ASSERT_EQ(getNameFromURL(parseURL("path:./nixpkgs#packages.x86_64-linux.complex^bin,man")), "complex");
+        ASSERT_EQ(getNameFromURL(parseURL("path:./myproj#packages.x86_64-linux.default^*")), "myproj");
 
         ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nixpkgs#packages.x86_64-linux.hello")), "hello");
         ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nixpkgs#hello")), "hello");
@@ -60,5 +62,6 @@ namespace nix {
         ASSERT_EQ(getNameFromURL(parseURL("path:.")), std::nullopt);
         ASSERT_EQ(getNameFromURL(parseURL("file:.#")), std::nullopt);
         ASSERT_EQ(getNameFromURL(parseURL("path:.#packages.x86_64-linux.default")), std::nullopt);
+        ASSERT_EQ(getNameFromURL(parseURL("path:.#packages.x86_64-linux.default^*")), std::nullopt);
     }
 }
diff --git a/src/libutil/url-name.cc b/src/libutil/url-name.cc
index ab65e78df..f94383e32 100644
--- a/src/libutil/url-name.cc
+++ b/src/libutil/url-name.cc
@@ -5,12 +5,13 @@
 namespace nix {
 
 static const std::string attributeNamePattern("[a-z0-9_-]+");
-static const std::regex lastAttributeRegex("(?:" + attributeNamePattern + "\\.)*(?!default)(" + attributeNamePattern +")");
+static const std::regex lastAttributeRegex("(?:" + attributeNamePattern + "\\.)*(?!default)(" + attributeNamePattern +")(\\^.*)?");
 static const std::string pathSegmentPattern("[a-zA-Z0-9_-]+");
 static const std::regex lastPathSegmentRegex(".*/(" + pathSegmentPattern +")");
 static const std::regex secondPathSegmentRegex("(?:" + pathSegmentPattern + ")/(" + pathSegmentPattern +")(?:/.*)?");
 static const std::regex gitProviderRegex("github|gitlab|sourcehut");
 static const std::regex gitSchemeRegex("git($|\\+.*)");
+static const std::regex defaultOutputRegex(".*\\.default($|\\^.*)");
 
 std::optional<std::string> getNameFromURL(ParsedURL url) {
   std::smatch match;
@@ -32,7 +33,7 @@ std::optional<std::string> getNameFromURL(ParsedURL url) {
     return match.str(1);
 
   /* If everything failed but there is a non-default fragment, use it in full */
-  if (!url.fragment.empty() && !hasSuffix(url.fragment, "default"))
+  if (!url.fragment.empty() && !std::regex_match(url.fragment, defaultOutputRegex))
     return url.fragment;
 
   /* If there is no fragment, take the last element of the path */
diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh
index 5c5a30dc2..59c17df34 100644
--- a/src/libutil/url-parts.hh
+++ b/src/libutil/url-parts.hh
@@ -19,6 +19,7 @@ const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncod
 const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?";
 const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])";
 const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*";
+const static std::string fragmentRegex = "(?:" + pcharRegex + "|[/? \"^])*";
 const static std::string segmentRegex = "(?:" + pcharRegex + "*)";
 const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)";
 const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)";
diff --git a/src/libutil/url.cc b/src/libutil/url.cc
index 9b438e6cd..2a0a5c839 100644
--- a/src/libutil/url.cc
+++ b/src/libutil/url.cc
@@ -15,7 +15,7 @@ ParsedURL parseURL(const std::string & url)
         "((" + schemeRegex + "):"
         + "(?:(?://(" + authorityRegex + ")(" + absPathRegex + "))|(/?" + pathRegex + ")))"
         + "(?:\\?(" + queryRegex + "))?"
-        + "(?:#(" + queryRegex + "))?",
+        + "(?:#(" + fragmentRegex + "))?",
         std::regex::ECMAScript);
 
     std::smatch match;
diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh
index 1fdbfb644..eced4d3f1 100644
--- a/tests/functional/nix-profile.sh
+++ b/tests/functional/nix-profile.sh
@@ -126,6 +126,7 @@ nix profile install "$flake1Dir^*"
 [ -e $TEST_HOME/.nix-profile/include ]
 
 printf Nix > $flake1Dir/who
+nix profile list
 nix profile upgrade flake1
 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Nix" ]]
 [ -e $TEST_HOME/.nix-profile/share/man ]

From a903f85f84b78a28490f3aa9615ba87d070d01d1 Mon Sep 17 00:00:00 2001
From: Artturin <Artturin@artturin.com>
Date: Tue, 31 Oct 2023 01:36:13 +0200
Subject: [PATCH 016/421] `nix-env --query`: fix `--json` ignoring `--drv-path`

```json
{
  "AMB-plugins": {
    "drvPath": "/nix/store/l99cb7h2hy8dg005arsjbd9kx0w05d3h-AMB-plugins-0.8.1.drv",
    "name": "AMB-plugins-0.8.1",
    "outputName": "out",
    "outputs": {
      "out": null
    },
    "pname": "AMB-plugins",
    "system": "x86_64-linux",
    "version": "0.8.1"
  },
  "ArchiSteamFarm": {
    "drvPath": "/nix/store/nhplgyjj34fz6hjmnyih25gxscfh8s7b-ArchiSteamFarm-5.4.12.5.drv",
    "name": "ArchiSteamFarm-5.4.12.5",
    "outputName": "out",
    "outputs": {
      "out": null
    },
    "pname": "ArchiSteamFarm",
    "system": "x86_64-linux",
    "version": "5.4.12.5"
  },
...
```
---
 doc/manual/src/release-notes/rl-next.md | 2 ++
 src/nix-env/nix-env.cc                  | 9 +++++++--
 tests/functional/user-envs.sh           | 1 +
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index 276252c37..2163a5392 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -12,6 +12,8 @@
 
 - Introduce a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash).
 
+- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.
+
 - `nix-shell` shebang lines now support single-quoted arguments.
 
 - `builtins.fetchTree` is now marked as stable.
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 01742daa8..558d0d6cd 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -922,7 +922,7 @@ static VersionDiff compareVersionAgainstSet(
 }
 
 
-static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool printOutPath, bool printMeta)
+static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool printOutPath, bool printDrvPath, bool printMeta)
 {
     using nlohmann::json;
     json topObj = json::object();
@@ -953,6 +953,11 @@ static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool prin
                 }
             }
 
+            if (printDrvPath) {
+                auto drvPath = i.queryDrvPath();
+                if (drvPath) pkgObj["drvPath"] = globals.state->store->printStorePath(*drvPath);
+            }
+
             if (printMeta) {
                 json &metaObj = pkgObj["meta"];
                 metaObj = json::object();
@@ -1079,7 +1084,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
 
     /* Print the desired columns, or XML output. */
     if (jsonOutput) {
-        queryJSON(globals, elems, printOutPath, printMeta);
+        queryJSON(globals, elems, printOutPath, printDrvPath, printMeta);
         cout << '\n';
         return;
     }
diff --git a/tests/functional/user-envs.sh b/tests/functional/user-envs.sh
index d1260ba04..dcd6b1b97 100644
--- a/tests/functional/user-envs.sh
+++ b/tests/functional/user-envs.sh
@@ -26,6 +26,7 @@ nix-env -f ./user-envs.nix -qa --json --out-path | jq -e '.[] | select(.name ==
     .outputName == "out",
     (.outputs.out | test("'$NIX_STORE_DIR'.*-0\\.1"))
 ] | all'
+nix-env -f ./user-envs.nix -qa --json --drv-path | jq -e '.[] | select(.name == "bar-0.1") | (.drvPath | test("'$NIX_STORE_DIR'.*-0\\.1\\.drv"))'
 
 # Query descriptions.
 nix-env -f ./user-envs.nix -qa '*' --description | grepQuiet silly

From cf59ea83ec98522113bf2fd81678537a871d0339 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 10 Nov 2023 13:58:59 +0100
Subject: [PATCH 017/421] configure: Check for libgit2

---
 Makefile.config.in       | 7 ++++---
 configure.ac             | 6 ++++++
 src/libfetchers/local.mk | 2 +-
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index 19992fa20..aadece0e1 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -8,7 +8,9 @@ CXX = @CXX@
 CXXFLAGS = @CXXFLAGS@
 CXXLTO = @CXXLTO@
 EDITLINE_LIBS = @EDITLINE_LIBS@
+ENABLE_BUILD = @ENABLE_BUILD@
 ENABLE_S3 = @ENABLE_S3@
+ENABLE_TESTS = @ENABLE_TESTS@
 GTEST_LIBS = @GTEST_LIBS@
 HAVE_LIBCPUID = @HAVE_LIBCPUID@
 HAVE_SECCOMP = @HAVE_SECCOMP@
@@ -17,6 +19,7 @@ LDFLAGS = @LDFLAGS@
 LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
 LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
 LIBCURL_LIBS = @LIBCURL_LIBS@
+LIBGIT2_LIBS = @LIBGIT2_LIBS@
 LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@
 LOWDOWN_LIBS = @LOWDOWN_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
@@ -35,6 +38,7 @@ docdir = @docdir@
 embedded_sandbox_shell = @embedded_sandbox_shell@
 exec_prefix = @exec_prefix@
 includedir = @includedir@
+internal_api_docs = @internal_api_docs@
 libdir = @libdir@
 libexecdir = @libexecdir@
 localstatedir = @localstatedir@
@@ -46,6 +50,3 @@ sandbox_shell = @sandbox_shell@
 storedir = @storedir@
 sysconfdir = @sysconfdir@
 system = @system@
-ENABLE_BUILD = @ENABLE_BUILD@
-ENABLE_TESTS = @ENABLE_TESTS@
-internal_api_docs = @internal_api_docs@
diff --git a/configure.ac b/configure.ac
index 75ce7d01d..1cda0852a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -335,9 +335,15 @@ AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation
   doc_generate=$enableval, doc_generate=yes)
 AC_SUBST(doc_generate)
 
+
 # Look for lowdown library.
 PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"])
 
+
+# Look for libgit2.
+PKG_CHECK_MODULES([LIBGIT2], [libgit2])
+
+
 # Setuid installations.
 AC_CHECK_FUNCS([setresuid setreuid lchown])
 
diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk
index f21651d77..266e7a211 100644
--- a/src/libfetchers/local.mk
+++ b/src/libfetchers/local.mk
@@ -8,6 +8,6 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc)
 
 libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
 
-libfetchers_LDFLAGS += -pthread -lgit2 -larchive
+libfetchers_LDFLAGS += -pthread $(LIBGIT2_LIBS) -larchive
 
 libfetchers_LIBS = libutil libstore

From e4cbdd26e0e6a2a5907dff8e60c3645f7d94423a Mon Sep 17 00:00:00 2001
From: Bouke van der Bijl <i@bou.ke>
Date: Mon, 13 Nov 2023 17:13:52 +0100
Subject: [PATCH 018/421] Add TODO comment for include try/catch

---
 src/libutil/config.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 17380b6d8..ab873b4a8 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -125,7 +125,9 @@ static void applyConfigInner(const std::string & contents, const std::string & p
                 try {
                     std::string includedContents = readFile(path);
                     applyConfigInner(includedContents, p, parsedContents);
-                } catch (SysError &) { }
+                } catch (SysError &) {
+                    // TODO: Do we actually want to ignore this? Or is it better to fail?
+                }
             } else if (!ignoreMissing) {
                 throw Error("file '%1%' included from '%2%' not found", p, path);
             }

From d6898cd58b1a685404ba6878c317e60be9473a9a Mon Sep 17 00:00:00 2001
From: Bouke van der Bijl <i@bou.ke>
Date: Mon, 13 Nov 2023 17:14:05 +0100
Subject: [PATCH 019/421] Move applyConfigFile to lambda inside libstore

---
 src/libstore/globals.cc | 11 +++++++++--
 src/libutil/config.cc   |  8 --------
 src/libutil/config.hh   |  6 ------
 3 files changed, 9 insertions(+), 16 deletions(-)

diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 9c25d9868..0aecd2b6a 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -111,7 +111,14 @@ Settings::Settings()
 
 void loadConfFile()
 {
-    globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf");
+    auto applyConfigFile = [&](const Path & path) {
+        try {
+            std::string contents = readFile(path);
+            globalConfig.applyConfig(contents, path);
+        } catch (SysError &) { }
+    };
+
+    applyConfigFile(settings.nixConfDir + "/nix.conf");
 
     /* We only want to send overrides to the daemon, i.e. stuff from
        ~/.nix/nix.conf or the command line. */
@@ -119,7 +126,7 @@ void loadConfFile()
 
     auto files = settings.nixUserConfFiles;
     for (auto file = files.rbegin(); file != files.rend(); file++) {
-        globalConfig.applyConfigFile(*file);
+        applyConfigFile(*file);
     }
 
     auto nixConfEnv = getEnv("NIX_CONFIG");
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index ab873b4a8..ad16c86bd 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -165,14 +165,6 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string
             set(name, value);
 }
 
-void AbstractConfig::applyConfigFile(const Path & path)
-{
-    try {
-        std::string contents = readFile(path);
-        applyConfig(contents, path);
-    } catch (SysError &) { }
-}
-
 void Config::resetOverridden()
 {
     for (auto & s : _settings)
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 38c3ce0c4..d49eb602d 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -82,12 +82,6 @@ public:
      */
     void applyConfig(const std::string & contents, const std::string & path = "<unknown>");
 
-    /**
-     * Applies a nix configuration file
-     * - path: the location of the config file to apply
-     */
-    void applyConfigFile(const Path & path);
-
     /**
      * Resets the `overridden` flag of all Settings
      */

From 21bb180547118e29a66bf091bd6b1dd911b3114d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 13:30:51 +0100
Subject: [PATCH 020/421] Use libgit2 with ssh-exec support

See https://github.com/libgit2/libgit2/pull/6617. This ensures that we
get support for ~/.ssh/config, known_hosts etc.
---
 flake.lock                   | 17 +++++++++++++++++
 flake.nix                    |  9 +++++++--
 src/libfetchers/git-utils.cc | 17 -----------------
 3 files changed, 24 insertions(+), 19 deletions(-)

diff --git a/flake.lock b/flake.lock
index 991cef1ee..2b1d96e4e 100644
--- a/flake.lock
+++ b/flake.lock
@@ -16,6 +16,22 @@
         "type": "github"
       }
     },
+    "libgit2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1697646580,
+        "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=",
+        "owner": "libgit2",
+        "repo": "libgit2",
+        "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5",
+        "type": "github"
+      },
+      "original": {
+        "owner": "libgit2",
+        "repo": "libgit2",
+        "type": "github"
+      }
+    },
     "lowdown-src": {
       "flake": false,
       "locked": {
@@ -67,6 +83,7 @@
     "root": {
       "inputs": {
         "flake-compat": "flake-compat",
+        "libgit2": "libgit2",
         "lowdown-src": "lowdown-src",
         "nixpkgs": "nixpkgs",
         "nixpkgs-regression": "nixpkgs-regression"
diff --git a/flake.nix b/flake.nix
index e71aa5374..d6a173081 100644
--- a/flake.nix
+++ b/flake.nix
@@ -7,8 +7,9 @@
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
+  inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; };
 
-  outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, flake-compat }:
+  outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, flake-compat, libgit2 }:
 
     let
       inherit (nixpkgs) lib;
@@ -194,7 +195,11 @@
             bzip2 xz brotli editline
             openssl sqlite
             libarchive
-            libgit2
+            (pkgs.libgit2.overrideAttrs (attrs: {
+              src = libgit2;
+              version = libgit2.lastModifiedDate;
+              cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"];
+            }))
             boost
             lowdown-nix
             libsodium
diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 3a0e2d02f..1ec50099b 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -336,9 +336,6 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         const std::string & url,
         const std::string & refspec) override
     {
-        /* FIXME: use libgit2. Unfortunately, it doesn't support
-           ssh_config at the moment. */
-        #if 0
         Remote remote;
 
         if (git_remote_create_anonymous(Setter(remote), *this, url.c_str()))
@@ -352,20 +349,6 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
         if (git_remote_fetch(remote.get(), &refspecs2, nullptr, nullptr))
             throw Error("fetching '%s' from '%s': %s", refspec, url, git_error_last()->message);
-        #endif
-
-        // FIXME: git stderr messes up our progress indicator, so
-        // we're using --quiet for now. Should process its stderr.
-        runProgram("git", true,
-            { "-C", path.abs(),
-              "--bare",
-              "fetch",
-              "--quiet",
-              "--force",
-              "--",
-              url,
-              refspec
-            }, {}, true);
     }
 
     void verifyCommit(

From d74d2fdaa721cd7cddceca2e0b4063a1d891bb9f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 13:35:26 +0100
Subject: [PATCH 021/421] Move statusCallbackTrampoline

---
 src/libfetchers/git-utils.cc | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 1ec50099b..ffcc92fc7 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -130,11 +130,6 @@ T peelObject(git_repository * repo, git_object * obj, git_object_t type)
     return obj2;
 }
 
-int statusCallbackTrampoline(const char * path, unsigned int statusFlags, void * payload)
-{
-    return (*((std::function<int(const char * path, unsigned int statusFlags)> *) payload))(path, statusFlags);
-}
-
 struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 {
     CanonPath path;
@@ -255,6 +250,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         return result;
     }
 
+    // Helper for statusCallback below.
+    static int statusCallbackTrampoline(const char * path, unsigned int statusFlags, void * payload)
+    {
+        return (*((std::function<int(const char * path, unsigned int statusFlags)> *) payload))(path, statusFlags);
+    }
+
     WorkdirInfo getWorkdirInfo() override
     {
         WorkdirInfo info;

From 38b07d63479ebdd4f43145264a026a22a72d940b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 13:38:03 +0100
Subject: [PATCH 022/421] src/libfetchers/git.cc: Apply suggestion

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 src/libfetchers/git.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 9c2a7df16..12233ed0a 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -332,7 +332,7 @@ struct GitInputScheme : InputScheme
            whether the working directory is dirty compared to HEAD. */
         GitRepo::WorkdirInfo workdirInfo;
 
-        /* URL of the repo, or its path if isLocal. */
+        /* URL of the repo, or its path if isLocal. Never a `file` URL. */
         std::string url;
 
         void warnDirty() const

From 25cf8f107125eda79e7faece90e7e05093a39e65 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 13:57:24 +0100
Subject: [PATCH 023/421] src/libfetchers/union-input-accessor.cc: Apply
 suggestion

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 src/libfetchers/union-input-accessor.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/libfetchers/union-input-accessor.cc b/src/libfetchers/union-input-accessor.cc
index ae942cb41..f9472efa7 100644
--- a/src/libfetchers/union-input-accessor.cc
+++ b/src/libfetchers/union-input-accessor.cc
@@ -12,8 +12,7 @@ struct UnionInputAccessor : InputAccessor
         // Currently we require a root filesystem. This could be relaxed.
         assert(mounts.contains(CanonPath::root));
 
-        // FIXME: should check that every mount point exists. Or we
-        // could return dummy parent directories automatically.
+        // FIXME: return dummy parent directories automatically?
     }
 
     std::string readFile(const CanonPath & path) override

From 4329bdf6a30fadad66384f0b8c835d7dba9f87b3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 13:58:27 +0100
Subject: [PATCH 024/421] Move comment

---
 src/libfetchers/cache.hh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh
index b517d496e..c8d3248bc 100644
--- a/src/libfetchers/cache.hh
+++ b/src/libfetchers/cache.hh
@@ -6,13 +6,14 @@
 
 namespace nix::fetchers {
 
+/*
+ * A cache for arbitrary `Attrs` -> `Attrs` mappings with a timestamp
+ * for expiration.
+ */
 struct Cache
 {
     virtual ~Cache() { }
 
-    /* A cache for arbitrary Attrs -> Attrs mappings with a timestamp
-       for expiration. */
-
     /*
      * Add a value to the cache. The cache is an arbitrary mapping of
      * Attrs to Attrs.

From 21140c987b7a301c01498864efbc3d92be04aced Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 13:59:00 +0100
Subject: [PATCH 025/421] Fix doxygen comments

---
 src/libfetchers/cache.hh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh
index c8d3248bc..f70589267 100644
--- a/src/libfetchers/cache.hh
+++ b/src/libfetchers/cache.hh
@@ -6,7 +6,7 @@
 
 namespace nix::fetchers {
 
-/*
+/**
  * A cache for arbitrary `Attrs` -> `Attrs` mappings with a timestamp
  * for expiration.
  */
@@ -14,7 +14,7 @@ struct Cache
 {
     virtual ~Cache() { }
 
-    /*
+    /**
      * Add a value to the cache. The cache is an arbitrary mapping of
      * Attrs to Attrs.
      */
@@ -22,13 +22,13 @@ struct Cache
         const Attrs & inAttrs,
         const Attrs & infoAttrs) = 0;
 
-    /*
+    /**
      * Look up a key with infinite TTL.
      */
     virtual std::optional<Attrs> lookup(
         const Attrs & inAttrs) = 0;
 
-    /*
+    /**
      * Look up a key. Return nothing if its TTL has exceeded
      * `settings.tarballTTL`.
      */
@@ -41,7 +41,7 @@ struct Cache
         Attrs infoAttrs;
     };
 
-    /*
+    /**
      * Look up a key. Return a bool denoting whether its TTL has
      * exceeded `settings.tarballTTL`.
      */

From 7f576f5dfe11c3f6b0e69179de95c921caddda18 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 14:01:38 +0100
Subject: [PATCH 026/421] Rename UnionInputAccessor to MountedInputAccessor

---
 src/libfetchers/git.cc                                 | 10 +++++-----
 ...ion-input-accessor.cc => mounted-input-accessor.cc} | 10 +++++-----
 src/libfetchers/mounted-input-accessor.hh              |  9 +++++++++
 src/libfetchers/union-input-accessor.hh                |  9 ---------
 4 files changed, 19 insertions(+), 19 deletions(-)
 rename src/libfetchers/{union-input-accessor.cc => mounted-input-accessor.cc} (86%)
 create mode 100644 src/libfetchers/mounted-input-accessor.hh
 delete mode 100644 src/libfetchers/union-input-accessor.hh

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 12233ed0a..90c6ad531 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -9,7 +9,7 @@
 #include "processes.hh"
 #include "git.hh"
 #include "fs-input-accessor.hh"
-#include "union-input-accessor.hh"
+#include "mounted-input-accessor.hh"
 #include "git-utils.hh"
 
 #include "fetch-settings.hh"
@@ -587,7 +587,7 @@ struct GitInputScheme : InputScheme
 
         auto accessor = repo->getAccessor(rev);
 
-        /* If the repo has submodules, fetch them and return a union
+        /* If the repo has submodules, fetch them and return a mounted
            input accessor consisting of the accessor for the top-level
            repo and the accessors for the submodules. */
         if (repoInfo.submodules) {
@@ -611,7 +611,7 @@ struct GitInputScheme : InputScheme
 
             if (!mounts.empty()) {
                 mounts.insert_or_assign(CanonPath::root, accessor);
-                accessor = makeUnionInputAccessor(std::move(mounts));
+                accessor = makeMountedInputAccessor(std::move(mounts));
             }
         }
 
@@ -636,7 +636,7 @@ struct GitInputScheme : InputScheme
         ref<InputAccessor> accessor =
             makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url));
 
-        /* If the repo has submodules, return a union input accessor
+        /* If the repo has submodules, return a mounted input accessor
            consisting of the accessor for the top-level repo and the
            accessors for the submodule workdirs. */
         if (repoInfo.submodules && !repoInfo.workdirInfo.submodules.empty()) {
@@ -660,7 +660,7 @@ struct GitInputScheme : InputScheme
             }
 
             mounts.insert_or_assign(CanonPath::root, accessor);
-            accessor = makeUnionInputAccessor(std::move(mounts));
+            accessor = makeMountedInputAccessor(std::move(mounts));
         }
 
         if (!repoInfo.workdirInfo.isDirty) {
diff --git a/src/libfetchers/union-input-accessor.cc b/src/libfetchers/mounted-input-accessor.cc
similarity index 86%
rename from src/libfetchers/union-input-accessor.cc
rename to src/libfetchers/mounted-input-accessor.cc
index f9472efa7..49917f6e5 100644
--- a/src/libfetchers/union-input-accessor.cc
+++ b/src/libfetchers/mounted-input-accessor.cc
@@ -1,12 +1,12 @@
-#include "union-input-accessor.hh"
+#include "mounted-input-accessor.hh"
 
 namespace nix {
 
-struct UnionInputAccessor : InputAccessor
+struct MountedInputAccessor : InputAccessor
 {
     std::map<CanonPath, ref<InputAccessor>> mounts;
 
-    UnionInputAccessor(std::map<CanonPath, ref<InputAccessor>> _mounts)
+    MountedInputAccessor(std::map<CanonPath, ref<InputAccessor>> _mounts)
         : mounts(std::move(_mounts))
     {
         // Currently we require a root filesystem. This could be relaxed.
@@ -71,9 +71,9 @@ struct UnionInputAccessor : InputAccessor
     }
 };
 
-ref<InputAccessor> makeUnionInputAccessor(std::map<CanonPath, ref<InputAccessor>> mounts)
+ref<InputAccessor> makeMountedInputAccessor(std::map<CanonPath, ref<InputAccessor>> mounts)
 {
-    return make_ref<UnionInputAccessor>(std::move(mounts));
+    return make_ref<MountedInputAccessor>(std::move(mounts));
 }
 
 }
diff --git a/src/libfetchers/mounted-input-accessor.hh b/src/libfetchers/mounted-input-accessor.hh
new file mode 100644
index 000000000..b557c5dad
--- /dev/null
+++ b/src/libfetchers/mounted-input-accessor.hh
@@ -0,0 +1,9 @@
+#pragma once
+
+#include "input-accessor.hh"
+
+namespace nix {
+
+ref<InputAccessor> makeMountedInputAccessor(std::map<CanonPath, ref<InputAccessor>> mounts);
+
+}
diff --git a/src/libfetchers/union-input-accessor.hh b/src/libfetchers/union-input-accessor.hh
deleted file mode 100644
index 6a1649c1d..000000000
--- a/src/libfetchers/union-input-accessor.hh
+++ /dev/null
@@ -1,9 +0,0 @@
-#pragma once
-
-#include "input-accessor.hh"
-
-namespace nix {
-
-ref<InputAccessor> makeUnionInputAccessor(std::map<CanonPath, ref<InputAccessor>> mounts);
-
-}

From c257c824475c92cdfda5daa027db334b6a0137f8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 14:47:17 +0100
Subject: [PATCH 027/421] Cleanup

---
 src/libfetchers/mounted-input-accessor.cc | 10 ++++------
 src/libutil/canon-path.cc                 |  7 +++++++
 src/libutil/canon-path.hh                 |  6 ++++++
 3 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/src/libfetchers/mounted-input-accessor.cc b/src/libfetchers/mounted-input-accessor.cc
index 49917f6e5..6f397eb17 100644
--- a/src/libfetchers/mounted-input-accessor.cc
+++ b/src/libfetchers/mounted-input-accessor.cc
@@ -54,18 +54,16 @@ struct MountedInputAccessor : InputAccessor
     std::pair<ref<InputAccessor>, CanonPath> resolve(CanonPath path)
     {
         // Find the nearest parent of `path` that is a mount point.
-        std::vector<std::string> ss;
+        std::vector<std::string> subpath;
         while (true) {
             auto i = mounts.find(path);
             if (i != mounts.end()) {
-                auto subpath = CanonPath::root;
-                for (auto j = ss.rbegin(); j != ss.rend(); ++j)
-                    subpath.push(*j);
-                return {i->second, std::move(subpath)};
+                std::reverse(subpath.begin(), subpath.end());
+                return {i->second, CanonPath(subpath)};
             }
 
             assert(!path.isRoot());
-            ss.push_back(std::string(*path.baseName()));
+            subpath.push_back(std::string(*path.baseName()));
             path.pop();
         }
     }
diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc
index f678fae94..1e465f1f6 100644
--- a/src/libutil/canon-path.cc
+++ b/src/libutil/canon-path.cc
@@ -13,6 +13,13 @@ CanonPath::CanonPath(std::string_view raw, const CanonPath & root)
     : path(absPath((Path) raw, root.abs()))
 { }
 
+CanonPath::CanonPath(const std::vector<std::string> & elems)
+    : path("/")
+{
+    for (auto & s : elems)
+        push(s);
+}
+
 CanonPath CanonPath::fromCwd(std::string_view path)
 {
     return CanonPath(unchecked_t(), absPath((Path) path));
diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh
index eefe05ed5..6d0519f4f 100644
--- a/src/libutil/canon-path.hh
+++ b/src/libutil/canon-path.hh
@@ -6,6 +6,7 @@
 #include <cassert>
 #include <iostream>
 #include <set>
+#include <vector>
 
 namespace nix {
 
@@ -46,6 +47,11 @@ public:
         : path(std::move(path))
     { }
 
+    /**
+     * Construct a canon path from a vector of elements.
+     */
+    CanonPath(const std::vector<std::string> & elems);
+
     static CanonPath fromCwd(std::string_view path = ".");
 
     static CanonPath root;

From 6ec6b8aa363f566a8da0d6959753efa452b152cc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 15:52:18 +0100
Subject: [PATCH 028/421] Improve git submodule error reporting

---
 src/libfetchers/fetchers.cc | 10 ++++++++++
 src/libfetchers/fetchers.hh |  2 ++
 src/libfetchers/git.cc      |  4 ++--
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 19e089aa8..c2513e076 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -220,6 +220,16 @@ std::pair<StorePath, Input> Input::fetch(ref<Store> store) const
     return {std::move(storePath), input};
 }
 
+std::pair<ref<InputAccessor>, Input> Input::getAccessor(ref<Store> store) const
+{
+    try {
+        return scheme->getAccessor(store, *this);
+    } catch (Error & e) {
+        e.addTrace({}, "while fetching the input '%s'", to_string());
+        throw;
+    }
+}
+
 Input Input::applyOverrides(
     std::optional<std::string> ref,
     std::optional<Hash> rev) const
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index 6db1615f2..ce5aa4c69 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -83,6 +83,8 @@ public:
      */
     std::pair<StorePath, Input> fetch(ref<Store> store) const;
 
+    std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store) const;
+
     Input applyOverrides(
         std::optional<std::string> ref,
         std::optional<Hash> rev) const;
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 90c6ad531..71ae74dde 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -605,7 +605,7 @@ struct GitInputScheme : InputScheme
                 attrs.insert_or_assign("rev", submoduleRev.gitRev());
                 auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
                 auto [submoduleAccessor, submoduleInput2] =
-                    submoduleInput.scheme->getAccessor(store, submoduleInput);
+                    submoduleInput.getAccessor(store);
                 mounts.insert_or_assign(submodule.path, submoduleAccessor);
             }
 
@@ -649,7 +649,7 @@ struct GitInputScheme : InputScheme
                 attrs.insert_or_assign("url", submodulePath.abs());
                 auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
                 auto [submoduleAccessor, submoduleInput2] =
-                    submoduleInput.scheme->getAccessor(store, submoduleInput);
+                    submoduleInput.getAccessor(store);
 
                 /* If the submodule is dirty, mark this repo dirty as
                    well. */

From 2964a9f562748cc698ee1f6ecf1e0da4e63211b9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 16:00:21 +0100
Subject: [PATCH 029/421] Fix relative submodule handling

Tested on

  nix flake prefetch 'git+https://github.com/blender/blender.git?rev=4ed8a360e956daf2591add4d3c9ec0719e2628fe&submodules=1'
---
 src/libfetchers/git-utils.cc | 12 ++++++++++--
 src/libfetchers/git-utils.hh |  4 +++-
 src/libfetchers/git.cc       |  2 +-
 src/libutil/url.cc           |  8 ++++++++
 src/libutil/url.hh           |  5 +++++
 5 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index ffcc92fc7..1edafbf33 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -308,13 +308,21 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
     std::vector<std::tuple<Submodule, Hash>> getSubmodules(const Hash & rev) override;
 
-    std::string resolveSubmoduleUrl(const std::string & url) override
+    std::string resolveSubmoduleUrl(
+        const std::string & url,
+        const std::string & base) override
     {
         git_buf buf = GIT_BUF_INIT;
         if (git_submodule_resolve_url(&buf, *this, url.c_str()))
             throw Error("resolving Git submodule URL '%s'", url);
         Finally cleanup = [&]() { git_buf_dispose(&buf); };
-        return buf.ptr;
+
+        std::string res(buf.ptr);
+
+        if (!hasPrefix(res, "/") && res.find("://") == res.npos)
+            res = parseURL(base + "/" + res).canonicalise().to_string();
+
+        return res;
     }
 
     bool hasObject(const Hash & oid_) override
diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh
index 7efbdedce..e0cb2c34f 100644
--- a/src/libfetchers/git-utils.hh
+++ b/src/libfetchers/git-utils.hh
@@ -59,7 +59,9 @@ struct GitRepo
      */
     virtual std::vector<std::tuple<Submodule, Hash>> getSubmodules(const Hash & rev) = 0;
 
-    virtual std::string resolveSubmoduleUrl(const std::string & url) = 0;
+    virtual std::string resolveSubmoduleUrl(
+        const std::string & url,
+        const std::string & base) = 0;
 
     struct TarballInfo
     {
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 71ae74dde..177c8b66e 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -594,7 +594,7 @@ struct GitInputScheme : InputScheme
             std::map<CanonPath, nix::ref<InputAccessor>> mounts;
 
             for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev)) {
-                auto resolved = repo->resolveSubmoduleUrl(submodule.url);
+                auto resolved = repo->resolveSubmoduleUrl(submodule.url, repoInfo.url);
                 debug("Git submodule %s: %s %s %s -> %s",
                     submodule.path, submodule.url, submodule.branch, submoduleRev.gitRev(), resolved);
                 fetchers::Attrs attrs;
diff --git a/src/libutil/url.cc b/src/libutil/url.cc
index 9b438e6cd..57b64d607 100644
--- a/src/libutil/url.cc
+++ b/src/libutil/url.cc
@@ -2,6 +2,7 @@
 #include "url-parts.hh"
 #include "util.hh"
 #include "split.hh"
+#include "canon-path.hh"
 
 namespace nix {
 
@@ -141,6 +142,13 @@ bool ParsedURL::operator ==(const ParsedURL & other) const
         && fragment == other.fragment;
 }
 
+ParsedURL ParsedURL::canonicalise()
+{
+    ParsedURL res(*this);
+    res.path = CanonPath(res.path).abs();
+    return res;
+}
+
 /**
  * Parse a URL scheme of the form '(applicationScheme\+)?transportScheme'
  * into a tuple '(applicationScheme, transportScheme)'
diff --git a/src/libutil/url.hh b/src/libutil/url.hh
index 26c2dcc28..833f54678 100644
--- a/src/libutil/url.hh
+++ b/src/libutil/url.hh
@@ -19,6 +19,11 @@ struct ParsedURL
     std::string to_string() const;
 
     bool operator ==(const ParsedURL & other) const;
+
+    /**
+     * Remove `.` and `..` path elements.
+     */
+    ParsedURL canonicalise();
 };
 
 MakeError(BadURL, Error);

From 28909999116781e194e2eb1646f3ccec005e774f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 15 Nov 2023 13:57:20 +0100
Subject: [PATCH 030/421] Show Git fetch progress

---
 src/libfetchers/git-utils.cc | 30 +++++++++++++++++++++++++++++-
 src/libfetchers/git.cc       |  2 --
 src/libmain/progress-bar.cc  |  8 ++++++++
 src/libutil/logging.hh       |  2 ++
 4 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 1edafbf33..b7ef05c10 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -3,6 +3,7 @@
 #include "cache.hh"
 #include "finally.hh"
 #include "processes.hh"
+#include "signals.hh"
 
 #include <boost/core/span.hpp>
 
@@ -341,10 +342,32 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
     ref<InputAccessor> getAccessor(const Hash & rev) override;
 
+    static int sidebandProgressCallback(const char * str, int len, void * payload)
+    {
+        auto act = (Activity *) payload;
+        act->result(resFetchStatus, trim(std::string_view(str, len)));
+        return _isInterrupted ? -1 : 0;
+    }
+
+    static int transferProgressCallback(const git_indexer_progress * stats, void * payload)
+    {
+        auto act = (Activity *) payload;
+        act->result(resFetchStatus,
+            fmt("%d/%d objects received, %d/%d deltas indexed, %.1f MiB",
+                stats->received_objects,
+                stats->total_objects,
+                stats->indexed_deltas,
+                stats->total_deltas,
+                stats->received_bytes / (1024.0 * 1024.0)));
+        return _isInterrupted ? -1 : 0;
+    }
+
     void fetch(
         const std::string & url,
         const std::string & refspec) override
     {
+        Activity act(*logger, lvlTalkative, actFetchTree, fmt("fetching Git repository '%s'", url));
+
         Remote remote;
 
         if (git_remote_create_anonymous(Setter(remote), *this, url.c_str()))
@@ -356,7 +379,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
             .count = 1
         };
 
-        if (git_remote_fetch(remote.get(), &refspecs2, nullptr, nullptr))
+        git_fetch_options opts = GIT_FETCH_OPTIONS_INIT;
+        opts.callbacks.payload = &act;
+        opts.callbacks.sideband_progress = sidebandProgressCallback;
+        opts.callbacks.transfer_progress = transferProgressCallback;
+
+        if (git_remote_fetch(remote.get(), &refspecs2, &opts, nullptr))
             throw Error("fetching '%s' from '%s': %s", refspec, url, git_error_last()->message);
     }
 
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 177c8b66e..3e7dcd8de 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -520,8 +520,6 @@ struct GitInputScheme : InputScheme
             }
 
             if (doFetch) {
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", repoInfo.url));
-
                 try {
                     auto fetchRef = repoInfo.allRefs
                         ? "refs/*"
diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc
index a7aee47c3..3aa012ee1 100644
--- a/src/libmain/progress-bar.cc
+++ b/src/libmain/progress-bar.cc
@@ -340,6 +340,14 @@ public:
             state->activitiesByType[type].expected += j;
             update(*state);
         }
+
+        else if (type == resFetchStatus) {
+            auto i = state->its.find(act);
+            assert(i != state->its.end());
+            ActInfo & actInfo = *i->second;
+            actInfo.lastLine = getS(fields, 0);
+            update(*state);
+        }
     }
 
     void update(State & state)
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 5aa6bee95..183f2d8e1 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -23,6 +23,7 @@ typedef enum {
     actQueryPathInfo = 109,
     actPostBuildHook = 110,
     actBuildWaiting = 111,
+    actFetchTree = 112,
 } ActivityType;
 
 typedef enum {
@@ -34,6 +35,7 @@ typedef enum {
     resProgress = 105,
     resSetExpected = 106,
     resPostBuildLogLine = 107,
+    resFetchStatus = 108,
 } ResultType;
 
 typedef uint64_t ActivityId;

From 5dd4ae86877cedaf70ea70d80b89c66b850bdc5a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 15 Nov 2023 14:08:34 +0100
Subject: [PATCH 031/421] Remove unused cacheType field

---
 src/libfetchers/git.cc | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 3e7dcd8de..b066b384c 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -323,8 +323,6 @@ struct GitInputScheme : InputScheme
         bool submodules = false;
         bool allRefs = false;
 
-        std::string cacheType;
-
         /* Whether this is a local, non-bare repository. */
         bool isLocal = false;
 
@@ -371,11 +369,6 @@ struct GitInputScheme : InputScheme
             .allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false)
         };
 
-        repoInfo.cacheType = "git";
-        if (repoInfo.shallow) repoInfo.cacheType += "-shallow";
-        if (repoInfo.submodules) repoInfo.cacheType += "-submodules";
-        if (repoInfo.allRefs) repoInfo.cacheType += "-all-refs";
-
         // file:// URIs are normally not cloned (but otherwise treated the
         // same as remote URIs, i.e. we don't use the working tree or
         // HEAD). Exception: If _NIX_FORCE_HTTP is set, or the repo is a bare git

From 7ab91e72387b96d1926f1b9c95b919020d4ba962 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 15 Nov 2023 14:43:30 +0100
Subject: [PATCH 032/421] Implement shallow fetching

---
 src/libfetchers/git-utils.cc |  4 ++-
 src/libfetchers/git-utils.hh |  3 ++-
 src/libfetchers/git.cc       | 48 +++++++++++++++++++-----------------
 3 files changed, 31 insertions(+), 24 deletions(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index b7ef05c10..f554dcc5f 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -364,7 +364,8 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
     void fetch(
         const std::string & url,
-        const std::string & refspec) override
+        const std::string & refspec,
+        bool shallow) override
     {
         Activity act(*logger, lvlTalkative, actFetchTree, fmt("fetching Git repository '%s'", url));
 
@@ -380,6 +381,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         };
 
         git_fetch_options opts = GIT_FETCH_OPTIONS_INIT;
+        opts.depth = shallow ? 1 : GIT_FETCH_DEPTH_FULL;
         opts.callbacks.payload = &act;
         opts.callbacks.sideband_progress = sidebandProgressCallback;
         opts.callbacks.transfer_progress = transferProgressCallback;
diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh
index e0cb2c34f..1def82071 100644
--- a/src/libfetchers/git-utils.hh
+++ b/src/libfetchers/git-utils.hh
@@ -75,7 +75,8 @@ struct GitRepo
 
     virtual void fetch(
         const std::string & url,
-        const std::string & refspec) = 0;
+        const std::string & refspec,
+        bool shallow) = 0;
 
     /**
      * Verify that commit `rev` is signed by one of the keys in
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index b066b384c..7208a0b6d 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -219,9 +219,6 @@ struct GitInputScheme : InputScheme
                 || name == "publicKeys")
                 experimentalFeatureSettings.require(Xp::VerifiedFetches);
 
-        maybeGetBoolAttr(attrs, "shallow");
-        maybeGetBoolAttr(attrs, "submodules");
-        maybeGetBoolAttr(attrs, "allRefs");
         maybeGetBoolAttr(attrs, "verifyCommit");
 
         if (auto ref = maybeGetStrAttr(attrs, "ref")) {
@@ -234,6 +231,9 @@ struct GitInputScheme : InputScheme
         auto url = fixGitURL(getStrAttr(attrs, "url"));
         parseURL(url);
         input.attrs["url"] = url;
+        getShallowAttr(input);
+        getSubmodulesAttr(input);
+        getAllRefsAttr(input);
         return input;
     }
 
@@ -243,8 +243,10 @@ struct GitInputScheme : InputScheme
         if (url.scheme != "git") url.scheme = "git+" + url.scheme;
         if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev());
         if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
-        if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false))
+        if (getShallowAttr(input))
             url.query.insert_or_assign("shallow", "1");
+        if (getSubmodulesAttr(input))
+            url.query.insert_or_assign("submodules", "1");
         if (maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(false))
             url.query.insert_or_assign("verifyCommit", "1");
         auto publicKeys = getPublicKeys(input.attrs);
@@ -319,10 +321,6 @@ struct GitInputScheme : InputScheme
 
     struct RepoInfo
     {
-        bool shallow = false;
-        bool submodules = false;
-        bool allRefs = false;
-
         /* Whether this is a local, non-bare repository. */
         bool isLocal = false;
 
@@ -347,11 +345,21 @@ struct GitInputScheme : InputScheme
         std::string gitDir = ".git";
     };
 
+    bool getShallowAttr(const Input & input) const
+    {
+        return maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
+    }
+
     bool getSubmodulesAttr(const Input & input) const
     {
         return maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
     }
 
+    bool getAllRefsAttr(const Input & input) const
+    {
+        return maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
+    }
+
     RepoInfo getRepoInfo(const Input & input) const
     {
         auto checkHashType = [&](const std::optional<Hash> & hash)
@@ -363,11 +371,7 @@ struct GitInputScheme : InputScheme
         if (auto rev = input.getRev())
             checkHashType(rev);
 
-        RepoInfo repoInfo {
-            .shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false),
-            .submodules = getSubmodulesAttr(input),
-            .allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false)
-        };
+        RepoInfo repoInfo;
 
         // file:// URIs are normally not cloned (but otherwise treated the
         // same as remote URIs, i.e. we don't use the working tree or
@@ -501,7 +505,7 @@ struct GitInputScheme : InputScheme
             if (auto rev = input.getRev()) {
                 doFetch = !repo->hasObject(*rev);
             } else {
-                if (repoInfo.allRefs) {
+                if (getAllRefsAttr(input)) {
                     doFetch = true;
                 } else {
                     /* If the local ref is older than ‘tarball-ttl’ seconds, do a
@@ -514,7 +518,7 @@ struct GitInputScheme : InputScheme
 
             if (doFetch) {
                 try {
-                    auto fetchRef = repoInfo.allRefs
+                    auto fetchRef = getAllRefsAttr(input)
                         ? "refs/*"
                         : ref.compare(0, 5, "refs/") == 0
                         ? ref
@@ -522,7 +526,7 @@ struct GitInputScheme : InputScheme
                         ? ref
                         : "refs/heads/" + ref;
 
-                    repo->fetch(repoInfo.url, fmt("%s:%s", fetchRef, fetchRef));
+                    repo->fetch(repoInfo.url, fmt("%s:%s", fetchRef, fetchRef), getShallowAttr(input));
                 } catch (Error & e) {
                     if (!pathExists(localRefFile)) throw;
                     logError(e.info());
@@ -556,7 +560,7 @@ struct GitInputScheme : InputScheme
 
         auto isShallow = repo->isShallow();
 
-        if (isShallow && !repoInfo.shallow)
+        if (isShallow && !getShallowAttr(input))
             throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", repoInfo.url);
 
         // FIXME: check whether rev is an ancestor of ref?
@@ -568,7 +572,7 @@ struct GitInputScheme : InputScheme
             {"lastModified", getLastModified(repoInfo, repoDir, rev)},
         });
 
-        if (!repoInfo.shallow)
+        if (!getShallowAttr(input))
             infoAttrs.insert_or_assign("revCount",
                 getRevCount(repoInfo, repoDir, rev));
 
@@ -581,7 +585,7 @@ struct GitInputScheme : InputScheme
         /* If the repo has submodules, fetch them and return a mounted
            input accessor consisting of the accessor for the top-level
            repo and the accessors for the submodules. */
-        if (repoInfo.submodules) {
+        if (getSubmodulesAttr(input)) {
             std::map<CanonPath, nix::ref<InputAccessor>> mounts;
 
             for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev)) {
@@ -607,7 +611,7 @@ struct GitInputScheme : InputScheme
         }
 
         assert(!origRev || origRev == rev);
-        if (!repoInfo.shallow)
+        if (!getShallowAttr(input))
             input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
         input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
 
@@ -619,7 +623,7 @@ struct GitInputScheme : InputScheme
         RepoInfo & repoInfo,
         Input && input) const
     {
-        if (repoInfo.submodules)
+        if (getSubmodulesAttr(input))
             /* Create mountpoints for the submodules. */
             for (auto & submodule : repoInfo.workdirInfo.submodules)
                 repoInfo.workdirInfo.files.insert(submodule.path);
@@ -630,7 +634,7 @@ struct GitInputScheme : InputScheme
         /* If the repo has submodules, return a mounted input accessor
            consisting of the accessor for the top-level repo and the
            accessors for the submodule workdirs. */
-        if (repoInfo.submodules && !repoInfo.workdirInfo.submodules.empty()) {
+        if (getSubmodulesAttr(input) && !repoInfo.workdirInfo.submodules.empty()) {
             std::map<CanonPath, nix::ref<InputAccessor>> mounts;
 
             for (auto & submodule : repoInfo.workdirInfo.submodules) {

From 8c54a01df5ee59e4acf151dba8077a9842e8bdc5 Mon Sep 17 00:00:00 2001
From: Bob van der Linden <bobvanderlinden@gmail.com>
Date: Mon, 13 Mar 2023 21:14:19 +0100
Subject: [PATCH 033/421] nix: develop: always force SHELL to chosen shell

SHELL was inherited from the system environment. This resulted in a new
shell being started, but with SHELL still referring to the system shell
and not the one used by nix-develop.

Applications like make, use SHELL to run commands, which meant that
top-level commands are run inside the nix-develop-shell, but
sub-commands are ran inside the system shell.

This setenv forces SHELL to always be set to the shell used by
nix-develop.
---
 src/nix/develop.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 38482ed42..4a561e52b 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -293,7 +293,6 @@ struct Common : InstallableCommand, MixProfile
         "NIX_LOG_FD",
         "NIX_REMOTE",
         "PPID",
-        "SHELL",
         "SHELLOPTS",
         "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt
         "TEMP",
@@ -643,6 +642,10 @@ struct CmdDevelop : Common, MixEnvironment
             ignoreException();
         }
 
+        // Override SHELL with the one chosen for this environment.
+        // This is to make sure the system shell doesn't leak into the build environment.
+        setenv("SHELL", shell.data(), 1);
+
         // If running a phase or single command, don't want an interactive shell running after
         // Ctrl-C, so don't pass --rcfile
         auto args = phase || !command.empty() ? Strings{std::string(baseNameOf(shell)), rcFilePath}

From ceab20d056a119317fb29eb0e06dfd0eb0b9d8ad Mon Sep 17 00:00:00 2001
From: Bob van der Linden <bobvanderlinden@gmail.com>
Date: Mon, 13 Nov 2023 22:04:34 +0100
Subject: [PATCH 034/421] nix: develop: add tests for interactive shell

---
 tests/functional/flakes/develop.sh | 75 ++++++++++++++++++++++++++++++
 tests/functional/local.mk          |  1 +
 2 files changed, 76 insertions(+)
 create mode 100644 tests/functional/flakes/develop.sh

diff --git a/tests/functional/flakes/develop.sh b/tests/functional/flakes/develop.sh
new file mode 100644
index 000000000..59f731239
--- /dev/null
+++ b/tests/functional/flakes/develop.sh
@@ -0,0 +1,75 @@
+source ../common.sh
+
+clearStore
+rm -rf $TEST_HOME/.cache $TEST_HOME/.config $TEST_HOME/.local
+
+# Create flake under test.
+cp ../shell-hello.nix ../config.nix $TEST_HOME/
+cat <<EOF >$TEST_HOME/flake.nix
+{
+    inputs.nixpkgs.url = "$TEST_HOME/nixpkgs";
+    outputs = {self, nixpkgs}: {
+      packages.$system.hello = (import ./config.nix).mkDerivation {
+        name = "hello";
+        outputs = [ "out" "dev" ];
+        meta.outputsToInstall = [ "out" ];
+        buildCommand = "";
+      };
+    };
+}
+EOF
+
+# Create fake nixpkgs flake.
+mkdir -p $TEST_HOME/nixpkgs
+cp ../config.nix ../shell.nix $TEST_HOME/nixpkgs
+cat <<EOF >$TEST_HOME/nixpkgs/flake.nix
+{
+    outputs = {self}: {
+      legacyPackages.$system.bashInteractive = (import ./shell.nix {}).bashInteractive;
+    };
+}
+EOF
+
+cd $TEST_HOME
+
+# Test whether `nix develop` passes through environment variables.
+[[ "$(
+    ENVVAR=a nix develop --no-write-lock-file .#hello <<EOF
+echo "\$ENVVAR"
+EOF
+)" = "a" ]]
+
+# Test whether `nix develop --ignore-environment` does _not_ pass through environment variables.
+[[ -z "$(
+    ENVVAR=a nix develop --ignore-environment --no-write-lock-file .#hello <<EOF
+echo "\$ENVVAR"
+EOF
+)" ]]
+
+# Determine the bashInteractive executable.
+nix build --no-write-lock-file './nixpkgs#bashInteractive' --out-link ./bash-interactive
+BASH_INTERACTIVE_EXECUTABLE="$PWD/bash-interactive/bin/bash"
+
+# Test whether `nix develop` uses nixpkgs#bashInteractive shell.
+[[ "$(
+    nix develop --no-write-lock-file .#hello <<EOF
+export PATH="$PATH"
+ps --pid $$ --format exe --no-headers
+EOF
+)" -ef "$BASH_INTERACTIVE_EXECUTABLE" ]]
+
+# Test whether `nix develop` sets `SHELL` to nixpkgs#bashInteractive shell.
+[[ "$(
+    SHELL=custom nix develop --no-write-lock-file .#hello <<EOF
+echo "\$SHELL"
+EOF
+)" -ef "$BASH_INTERACTIVE_EXECUTABLE" ]]
+
+# Test whether `nix develop` with ignore environment sets `SHELL` to nixpkgs#bashInteractive shell.
+[[ "$(
+    SHELL=custom nix develop --ignore-environment --no-write-lock-file .#hello <<EOF
+echo "\$SHELL"
+EOF
+)" -ef "$BASH_INTERACTIVE_EXECUTABLE" ]]
+
+clearStore
diff --git a/tests/functional/local.mk b/tests/functional/local.mk
index 21dabca88..e6f16be43 100644
--- a/tests/functional/local.mk
+++ b/tests/functional/local.mk
@@ -2,6 +2,7 @@ nix_tests = \
   test-infra.sh \
   init.sh \
   flakes/flakes.sh \
+  flakes/develop.sh \
   flakes/run.sh \
   flakes/mercurial.sh \
   flakes/circular.sh \

From eaa36295328ecb4fad5be72b125186f4d4feee84 Mon Sep 17 00:00:00 2001
From: Bob van der Linden <bobvanderlinden@gmail.com>
Date: Thu, 16 Nov 2023 15:12:31 +0100
Subject: [PATCH 035/421] fixup! nix: develop: add tests for interactive shell

---
 tests/functional/common/vars-and-functions.sh.in | 1 +
 tests/functional/flakes/develop.sh               | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/functional/common/vars-and-functions.sh.in b/tests/functional/common/vars-and-functions.sh.in
index 848988af9..02773bf60 100644
--- a/tests/functional/common/vars-and-functions.sh.in
+++ b/tests/functional/common/vars-and-functions.sh.in
@@ -45,6 +45,7 @@ if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then
   DAEMON_PATH="${NIX_DAEMON_PACKAGE}/bin:$DAEMON_PATH"
 fi
 coreutils=@coreutils@
+lsof=@lsof@
 
 export dot=@dot@
 export SHELL="@bash@"
diff --git a/tests/functional/flakes/develop.sh b/tests/functional/flakes/develop.sh
index 59f731239..db23ca0c0 100644
--- a/tests/functional/flakes/develop.sh
+++ b/tests/functional/flakes/develop.sh
@@ -54,7 +54,7 @@ BASH_INTERACTIVE_EXECUTABLE="$PWD/bash-interactive/bin/bash"
 [[ "$(
     nix develop --no-write-lock-file .#hello <<EOF
 export PATH="$PATH"
-ps --pid $$ --format exe --no-headers
+$lsof -a -d txt -p \$$ 2>&1 | grep -o '/.*/bash'
 EOF
 )" -ef "$BASH_INTERACTIVE_EXECUTABLE" ]]
 

From 2eb59c34b531f03a85f67b9246ccaf0ff5fcad23 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 12 Nov 2023 03:02:02 +0100
Subject: [PATCH 036/421] Value: extract Value::StringWithContext

---
 src/libexpr/value.hh | 54 +++++++++++++++++++++++---------------------
 1 file changed, 28 insertions(+), 26 deletions(-)

diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 191cc30ba..0f8cef418 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -158,37 +158,39 @@ public:
     inline bool isPrimOp() const { return internalType == tPrimOp; };
     inline bool isPrimOpApp() const { return internalType == tPrimOpApp; };
 
+    /**
+     * Strings in the evaluator carry a so-called `context` which
+     * is a list of strings representing store paths.  This is to
+     * allow users to write things like
+     *
+     *   "--with-freetype2-library=" + freetype + "/lib"
+     *
+     * where `freetype` is a derivation (or a source to be copied
+     * to the store).  If we just concatenated the strings without
+     * keeping track of the referenced store paths, then if the
+     * string is used as a derivation attribute, the derivation
+     * will not have the correct dependencies in its inputDrvs and
+     * inputSrcs.
+
+     * The semantics of the context is as follows: when a string
+     * with context C is used as a derivation attribute, then the
+     * derivations in C will be added to the inputDrvs of the
+     * derivation, and the other store paths in C will be added to
+     * the inputSrcs of the derivations.
+
+     * For canonicity, the store paths should be in sorted order.
+     */
+    struct StringWithContext {
+        const char * c_str;
+        const char * * context; // must be in sorted order
+    };
+
     union
     {
         NixInt integer;
         bool boolean;
 
-        /**
-         * Strings in the evaluator carry a so-called `context` which
-         * is a list of strings representing store paths.  This is to
-         * allow users to write things like
-
-         *   "--with-freetype2-library=" + freetype + "/lib"
-
-         * where `freetype` is a derivation (or a source to be copied
-         * to the store).  If we just concatenated the strings without
-         * keeping track of the referenced store paths, then if the
-         * string is used as a derivation attribute, the derivation
-         * will not have the correct dependencies in its inputDrvs and
-         * inputSrcs.
-
-         * The semantics of the context is as follows: when a string
-         * with context C is used as a derivation attribute, then the
-         * derivations in C will be added to the inputDrvs of the
-         * derivation, and the other store paths in C will be added to
-         * the inputSrcs of the derivations.
-
-         * For canonicity, the store paths should be in sorted order.
-         */
-        struct {
-            const char * c_str;
-            const char * * context; // must be in sorted order
-        } string;
+        StringWithContext string;
 
         struct {
             InputAccessor * accessor;

From d8ff5cfe8eba34c8b4b5cc53f3b40cd3dfd84224 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 12 Nov 2023 03:03:37 +0100
Subject: [PATCH 037/421] Value: extract Value::Path

---
 src/libexpr/value.hh | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 0f8cef418..34f994997 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -185,6 +185,11 @@ public:
         const char * * context; // must be in sorted order
     };
 
+    struct Path {
+        InputAccessor * accessor;
+        const char * path;
+    };
+
     union
     {
         NixInt integer;
@@ -192,10 +197,7 @@ public:
 
         StringWithContext string;
 
-        struct {
-            InputAccessor * accessor;
-            const char * path;
-        } _path;
+        Path _path;
 
         Bindings * attrs;
         struct {

From b55203e874f8e4b2fc5289129efba791937c23d0 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 12 Nov 2023 03:06:04 +0100
Subject: [PATCH 038/421] Value: extract Value::ClosureThunk

---
 src/libexpr/value.hh | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 34f994997..4c51c52e4 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -190,6 +190,11 @@ public:
         const char * path;
     };
 
+    struct ClosureThunk {
+        Env * env;
+        Expr * expr;
+    };
+
     union
     {
         NixInt integer;
@@ -205,10 +210,7 @@ public:
             Value * * elems;
         } bigList;
         Value * smallList[2];
-        struct {
-            Env * env;
-            Expr * expr;
-        } thunk;
+        ClosureThunk thunk;
         struct {
             Value * left, * right;
         } app;

From 6af1d9f7b94da454252b62f0cfff4ce800c5a46b Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 12 Nov 2023 03:06:55 +0100
Subject: [PATCH 039/421] Value: extract Value::FunctionApplicationThunk

---
 src/libexpr/value.hh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 4c51c52e4..cfb3f5276 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -195,6 +195,10 @@ public:
         Expr * expr;
     };
 
+    struct FunctionApplicationThunk {
+        Value * left, * right;
+    };
+
     union
     {
         NixInt integer;
@@ -211,17 +215,13 @@ public:
         } bigList;
         Value * smallList[2];
         ClosureThunk thunk;
-        struct {
-            Value * left, * right;
-        } app;
+        FunctionApplicationThunk app;
         struct {
             Env * env;
             ExprLambda * fun;
         } lambda;
         PrimOp * primOp;
-        struct {
-            Value * left, * right;
-        } primOpApp;
+        FunctionApplicationThunk primOpApp;
         ExternalValueBase * external;
         NixFloat fpoint;
     };

From 7055c6528532fdd3b0ce9b8f5282b002fc011470 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 12 Nov 2023 03:07:32 +0100
Subject: [PATCH 040/421] Value: extract Value::Lambda

---
 src/libexpr/value.hh | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index cfb3f5276..93ccdbc2e 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -199,6 +199,11 @@ public:
         Value * left, * right;
     };
 
+    struct Lambda {
+        Env * env;
+        ExprLambda * fun;
+    };
+
     union
     {
         NixInt integer;
@@ -216,10 +221,7 @@ public:
         Value * smallList[2];
         ClosureThunk thunk;
         FunctionApplicationThunk app;
-        struct {
-            Env * env;
-            ExprLambda * fun;
-        } lambda;
+        Lambda lambda;
         PrimOp * primOp;
         FunctionApplicationThunk primOpApp;
         ExternalValueBase * external;

From 260c6147625e95e4772ccdee80d6463d242c7b64 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 12 Nov 2023 03:31:45 +0100
Subject: [PATCH 041/421] Value: use std::span, change use of const

**`Value` and `const`**

These two deserve some explanation. We'll get to lists later.

Values can normally be thought of as immutable, except they are
are also the vehicle for call by need, which must be implemented
using mutation.

This circumstance makes a `const Value` a rather useless thing:

 - If it's a thunk, you can't evaluate it, except by copying, but
   that would not be call by need.

 - If it's not a thunk, you know the type, so the method that
   acquired it for you should have returned something more specific,
   such as a `const Bindings &` (which actually does make sense
   because that's an immutable span of pointers to mutable `Value`s.

 - If you don't care about the type yet, you might establish the
   convention that `const Value` means `deepSeq`-ed data, but
   this is hardly useful and not actually as safe as you would
   supposedly want to trust it to be - just convention.

**Lists**

`std::span` is a tuple of pointer and size - just what we need.

We don't return them as `const Value`, because considering the
first bullet point we discussed before, we'd have to force all
the list values, which isn't what we want.

So what we end up with is a nice representation of a list in
weak head normal form: the spine is immutable, but the
items may need some evaluation later.
---
 src/libexpr/value.hh | 37 ++++++++-----------------------------
 1 file changed, 8 insertions(+), 29 deletions(-)

diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 93ccdbc2e..bcff8ae55 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -3,6 +3,7 @@
 
 #include <cassert>
 #include <climits>
+#include <span>
 
 #include "symbol-table.hh"
 #include "value/context.hh"
@@ -395,7 +396,13 @@ public:
         return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems;
     }
 
-    const Value * const * listElems() const
+    std::span<Value * const> listItems() const
+    {
+        assert(isList());
+        return std::span<Value * const>(listElems(), listSize());
+    }
+
+    Value * const * listElems() const
     {
         return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems;
     }
@@ -414,34 +421,6 @@ public:
      */
     bool isTrivial() const;
 
-    auto listItems()
-    {
-        struct ListIterable
-        {
-            typedef Value * const * iterator;
-            iterator _begin, _end;
-            iterator begin() const { return _begin; }
-            iterator end() const { return _end; }
-        };
-        assert(isList());
-        auto begin = listElems();
-        return ListIterable { begin, begin + listSize() };
-    }
-
-    auto listItems() const
-    {
-        struct ConstListIterable
-        {
-            typedef const Value * const * iterator;
-            iterator _begin, _end;
-            iterator begin() const { return _begin; }
-            iterator end() const { return _end; }
-        };
-        assert(isList());
-        auto begin = listElems();
-        return ConstListIterable { begin, begin + listSize() };
-    }
-
     SourcePath path() const
     {
         assert(internalType == tPath);

From 121665f3773bc46ca6df0dda6f66b1a86e7d9e72 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 12 Nov 2023 17:51:09 +0100
Subject: [PATCH 042/421] nix-env: Use state.mkList, required for correct stats

---
 src/nix-env/nix-env.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 213a20d93..ab1d8f713 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -172,7 +172,7 @@ static void loadSourceExpr(EvalState & state, const SourcePath & path, Value & v
        directory). */
     else if (st.type == InputAccessor::tDirectory) {
         auto attrs = state.buildBindings(maxAttrs);
-        attrs.alloc("_combineChannels").mkList(0);
+        state.mkList(attrs.alloc("_combineChannels"), 0);
         StringSet seen;
         getAllExprs(state, path, seen, attrs);
         v.mkAttrs(attrs);

From 251fb23aeab8f85afb4f8376c2e6fc3d8b229d23 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 19 Nov 2023 01:46:48 +0100
Subject: [PATCH 043/421] Shebang parser: add virtual destructor

Fixes:

    warning: destructor called on non-final 'nix::ParseUnquoted' that has virtual functions but non-virtual destructor [-Wdelete-non-abstract-non-virtual-dtor]
---
 src/libutil/args.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 4359c5e8e..4480a03f5 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -97,6 +97,8 @@ struct Parser {
     virtual void operator()(std::shared_ptr<Parser> & state, Strings & r) = 0;
 
     Parser(std::string_view s) : remaining(s) {};
+
+    virtual ~Parser() { };
 };
 
 struct ParseQuoted : public Parser {

From 70ddf298e0882075dcb1cf69562c629a195718f7 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <silvan.mosberger@tweag.io>
Date: Sun, 19 Nov 2023 04:09:14 +0100
Subject: [PATCH 044/421] doc: Add link to filterSource from path

---
 src/libexpr/primops.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index e274c3c0c..dda409955 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -2374,7 +2374,7 @@ static RegisterPrimOp primop_path({
           like `@`.
 
         - filter\
-          A function of the type expected by `builtins.filterSource`,
+          A function of the type expected by [`builtins.filterSource`](#builtins-filterSource),
           with the same semantics.
 
         - recursive\

From fe4f573d49a5c47cf9ffd0bd3fe8868104550818 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 19 Nov 2023 11:38:47 +0100
Subject: [PATCH 045/421] flake.nix: Update nixpkgs: release-23.05 ->
 nixos-23.05-small
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/9eb24edd6a0027fed010ccfe300a9734d029983c' (2023-11-01)
  → 'github:NixOS/nixpkgs/decdf666c833a325cb4417041a90681499e06a41' (2023-11-18)
---
 flake.lock | 8 ++++----
 flake.nix  | 4 +---
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/flake.lock b/flake.lock
index 991cef1ee..825166717 100644
--- a/flake.lock
+++ b/flake.lock
@@ -34,16 +34,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1698876495,
-        "narHash": "sha256-nsQo2/mkDUFeAjuu92p0dEqhRvHHiENhkKVIV1y0/Oo=",
+        "lastModified": 1700342017,
+        "narHash": "sha256-HaibwlWH5LuqsaibW3sIVjZQtEM/jWtOHX4Nk93abGE=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "9eb24edd6a0027fed010ccfe300a9734d029983c",
+        "rev": "decdf666c833a325cb4417041a90681499e06a41",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "release-23.05",
+        "ref": "nixos-23.05-small",
         "repo": "nixpkgs",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index 05ab7b06d..f21b1a63f 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,9 +1,7 @@
 {
   description = "The purely functional package manager";
 
-  # FIXME go back to nixos-23.05-small once
-  # https://github.com/NixOS/nixpkgs/pull/264875 is included.
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.05";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };

From 796a7eb92d2b0caf75685126adc7460a4c39cfec Mon Sep 17 00:00:00 2001
From: DavHau <hsngrmpf+github@gmail.com>
Date: Sun, 19 Nov 2023 20:32:23 +0700
Subject: [PATCH 046/421] fetchTree: clarify docs for shallow flag

---
 src/libexpr/primops/fetchTree.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 8031bf809..383ec7c58 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -425,7 +425,8 @@ static RegisterPrimOp primop_fetchGit({
 
       - `shallow` (default: `false`)
 
-        A Boolean parameter that specifies whether fetching a shallow clone is allowed.
+        A Boolean parameter that specifies whether fetching from a shallow remote repository is allowed.
+        This still performs a full clone of what is available on the remote.
 
       - `allRefs`
 

From d5928085d5a542b19cc21b1f299392ee7a0c960b Mon Sep 17 00:00:00 2001
From: nicoo <nicoo@mur.at>
Date: Sun, 19 Nov 2023 19:57:07 +0100
Subject: [PATCH 047/421] builtins.concatMap: Fix typo in error message

---
 src/libexpr/primops.cc            | 2 +-
 src/libexpr/tests/error_traces.cc | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index dda409955..27f502830 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -3459,7 +3459,7 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args,
     for (unsigned int n = 0; n < nrLists; ++n) {
         Value * vElem = args[1]->listElems()[n];
         state.callFunction(*args[0], *vElem, lists[n], pos);
-        state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)), "while evaluating the return value of the function passed to buitlins.concatMap");
+        state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)), "while evaluating the return value of the function passed to builtins.concatMap");
         len += lists[n].listSize();
     }
 
diff --git a/src/libexpr/tests/error_traces.cc b/src/libexpr/tests/error_traces.cc
index 139366bcd..81498f65a 100644
--- a/src/libexpr/tests/error_traces.cc
+++ b/src/libexpr/tests/error_traces.cc
@@ -906,12 +906,12 @@ namespace nix {
         ASSERT_TRACE2("concatMap (x: 1) [ \"foo\" ] # TODO",
                       TypeError,
                       hintfmt("value is %s while a list was expected", "an integer"),
-                      hintfmt("while evaluating the return value of the function passed to buitlins.concatMap"));
+                      hintfmt("while evaluating the return value of the function passed to builtins.concatMap"));
 
         ASSERT_TRACE2("concatMap (x: \"foo\") [ 1 2 ] # TODO",
                       TypeError,
                       hintfmt("value is %s while a list was expected", "a string"),
-                      hintfmt("while evaluating the return value of the function passed to buitlins.concatMap"));
+                      hintfmt("while evaluating the return value of the function passed to builtins.concatMap"));
 
     }
 

From 19993398a12069a868b0fb10b63f7d06f0f993e6 Mon Sep 17 00:00:00 2001
From: Qyriad <qyriad@qyriad.me>
Date: Mon, 20 Nov 2023 03:37:02 -0700
Subject: [PATCH 048/421] flakes: check for flake.nix before complaining that
 lstat on it fails

getFlake currently calls lstat (via isLink via canonPath) before it
performs the sanity check that a flake.nix exists in the first place.
This commit moves the check to before path canonicalization, so that
failed symlink check operations don't throw before the check does.
---
 src/libexpr/flake/flake.cc | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 54de53e0b..b128de31e 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -212,8 +212,16 @@ static Flake getFlake(
     auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree(
         state, originalRef, allowLookup, flakeCache);
 
+    // We need to guard against symlink attacks, but before we start doing
+    // filesystem operations we should make sure there's a flake.nix in the
+    // first place.
+    auto unsafeFlakeDir = state.store->toRealPath(storePath) + "/" + lockedRef.subdir;
+    auto unsafeFlakeFile = unsafeFlakeDir + "/flake.nix";
+    if (!pathExists(unsafeFlakeFile))
+        throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir);
+
     // Guard against symlink attacks.
-    auto flakeDir = canonPath(state.store->toRealPath(storePath) + "/" + lockedRef.subdir, true);
+    auto flakeDir = canonPath(unsafeFlakeDir, true);
     auto flakeFile = canonPath(flakeDir + "/flake.nix", true);
     if (!isInDir(flakeFile, state.store->toRealPath(storePath)))
         throw Error("'flake.nix' file of flake '%s' escapes from '%s'",
@@ -226,9 +234,6 @@ static Flake getFlake(
         .storePath = storePath,
     };
 
-    if (!pathExists(flakeFile))
-        throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir);
-
     Value vInfo;
     state.evalFile(state.rootPath(CanonPath(flakeFile)), vInfo, true); // FIXME: symlink attack
 

From 1d6abec993a371091459d5e23f985c6d69621ce7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Nov 2023 12:35:35 +0100
Subject: [PATCH 049/421] Revert use of boost::container::small_vector in the
 evaluator

It caused random crashes (https://hydra.nixos.org/build/241514506,
https://hydra.nixos.org/build/241443330) because the heap allocation
done by small_vector in the not-small case is not scanned for GC
roots.
---
 src/libexpr/eval.cc    | 11 +++++------
 src/libexpr/primops.cc |  7 ++++---
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index e9b8cacfd..46a49c891 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -31,7 +31,6 @@
 
 #include <sys/resource.h>
 #include <nlohmann/json.hpp>
-#include <boost/container/small_vector.hpp>
 
 #if HAVE_BOEHMGC
 
@@ -1710,7 +1709,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 /* We have all the arguments, so call the primop with
                    the previous and new arguments. */
 
-                Value * vArgs[maxPrimOpArity];
+                Value * vArgs[arity];
                 auto n = argsDone;
                 for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left)
                     vArgs[--n] = arg->primOpApp.right;
@@ -1773,11 +1772,11 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v)
     // 4: about 60
     // 5: under 10
     // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total.
-    boost::container::small_vector<Value *, 4> vArgs(args.size());
+    Value * vArgs[args.size()];
     for (size_t i = 0; i < args.size(); ++i)
         vArgs[i] = args[i]->maybeThunk(state, env);
 
-    state.callFunction(vFun, args.size(), vArgs.data(), v, pos);
+    state.callFunction(vFun, args.size(), vArgs, v, pos);
 }
 
 
@@ -2016,8 +2015,8 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
         return result;
     };
 
-    boost::container::small_vector<Value, conservativeStackReservation> values(es->size());
-    Value * vTmpP = values.data();
+    Value values[es->size()];
+    Value * vTmpP = values;
 
     for (auto & [i_pos, i] : *es) {
         Value & vTmp = *vTmpP++;
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 27f502830..a8d44d8b7 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -2729,7 +2729,7 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V
     auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs"));
     state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs");
 
-    boost::container::small_vector<Value *, nonRecursiveStackReservation> res(args[1]->listSize());
+    Value * res[args[1]->listSize()];
     size_t found = 0;
 
     for (auto v2 : args[1]->listItems()) {
@@ -3064,7 +3064,8 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val
 
     state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter");
 
-    boost::container::small_vector<Value *, nonRecursiveStackReservation> vs(args[1]->listSize());
+    // FIXME: putting this on the stack is risky.
+    Value * vs[args[1]->listSize()];
     size_t k = 0;
 
     bool same = true;
@@ -3453,7 +3454,7 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args,
     state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap");
     auto nrLists = args[1]->listSize();
 
-    boost::container::small_vector<Value, conservativeStackReservation> lists(nrLists);
+    Value lists[nrLists];
     size_t len = 0;
 
     for (unsigned int n = 0; n < nrLists; ++n) {

From 7ac39ff05c8353c665174e8df61dd76a2b0b93db Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 20 Nov 2023 13:11:58 +0100
Subject: [PATCH 050/421] refactor Store::buildPaths: convert to string earlier

Preparation for RFC 92 dynamic derivations.
---
 src/libstore/build/entry-points.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc
index 13ff22f45..74eca63f3 100644
--- a/src/libstore/build/entry-points.cc
+++ b/src/libstore/build/entry-points.cc
@@ -15,7 +15,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
 
     worker.run(goals);
 
-    StorePathSet failed;
+    StringSet failed;
     std::optional<Error> ex;
     for (auto & i : goals) {
         if (i->ex) {
@@ -26,9 +26,9 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
         }
         if (i->exitCode != Goal::ecSuccess) {
             if (auto i2 = dynamic_cast<DerivationGoal *>(i.get()))
-                failed.insert(i2->drvPath);
+                failed.insert(std::string { i2->drvPath.to_string() });
             else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
-                failed.insert(i2->storePath);
+                failed.insert(std::string { i2->storePath.to_string()});
         }
     }
 
@@ -37,7 +37,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
         throw std::move(*ex);
     } else if (!failed.empty()) {
         if (ex) logError(ex->info());
-        throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed));
+        throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed)));
     }
 }
 

From a5e51a9e02efe2813170fdf0093c98b3d56aed84 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 20 Nov 2023 13:18:25 +0100
Subject: [PATCH 051/421] refactor Worker::childStarted/Terminated: use switch

Preparation for RFC 92 dynamic derivations.
---
 src/libstore/build/worker.cc | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 01914e2d6..01f52e7ab 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -199,8 +199,16 @@ void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
     child.respectTimeouts = respectTimeouts;
     children.emplace_back(child);
     if (inBuildSlot) {
-        if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++;
-        else nrLocalBuilds++;
+        switch (goal->jobCategory()) {
+        case JobCategory::Substitution:
+            nrSubstitutions++;
+            break;
+        case JobCategory::Build:
+            nrLocalBuilds++;
+            break;
+        default:
+            abort();
+        }
     }
 }
 
@@ -212,12 +220,17 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
     if (i == children.end()) return;
 
     if (i->inBuildSlot) {
-        if (goal->jobCategory() == JobCategory::Substitution) {
+        switch (goal->jobCategory()) {
+        case JobCategory::Substitution:
             assert(nrSubstitutions > 0);
             nrSubstitutions--;
-        } else {
+            break;
+        case JobCategory::Build:
             assert(nrLocalBuilds > 0);
             nrLocalBuilds--;
+            break;
+        default:
+            abort();
         }
     }
 

From 2a96445d7505cb0a82ed2a49c7210b3073ffd153 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Nov 2023 15:06:28 +0100
Subject: [PATCH 052/421] Bump version

---
 .version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.version b/.version
index ef0f38abe..7329e21c3 100644
--- a/.version
+++ b/.version
@@ -1 +1 @@
-2.19.0
+2.20.0

From e2b6821ca0147f36bcb9404aab080f80746984c8 Mon Sep 17 00:00:00 2001
From: roblabla <unfiltered@roblab.la>
Date: Mon, 20 Nov 2023 15:41:38 +0100
Subject: [PATCH 053/421] Fix bad_format_string error when builder stdout
 contains %

---
 src/libutil/serialise.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index d7950b11b..f465bd0de 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -448,7 +448,7 @@ Error readError(Source & source)
     auto msg = readString(source);
     ErrorInfo info {
         .level = level,
-        .msg = hintformat(fmt("%s", msg)),
+        .msg = hintfmt(msg),
     };
     auto havePos = readNum<size_t>(source);
     assert(havePos == 0);
@@ -457,7 +457,7 @@ Error readError(Source & source)
         havePos = readNum<size_t>(source);
         assert(havePos == 0);
         info.traces.push_back(Trace {
-            .hint = hintformat(fmt("%s", readString(source)))
+            .hint = hintfmt(readString(source))
         });
     }
     return Error(std::move(info));

From e4066c04442f86a5a12d492d588e3e82b533053d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 15 Nov 2023 16:15:47 +0100
Subject: [PATCH 054/421] Fetch specific Git revisions

This is more efficient, and necessary when using shallow=1 with a rev.
---
 src/libfetchers/git.cc       | 5 ++++-
 tests/functional/fetchGit.sh | 4 +---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 7208a0b6d..2fd3fb41e 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -518,8 +518,11 @@ struct GitInputScheme : InputScheme
 
             if (doFetch) {
                 try {
-                    auto fetchRef = getAllRefsAttr(input)
+                    auto fetchRef =
+                        getAllRefsAttr(input)
                         ? "refs/*"
+                        : input.getRev()
+                        ? input.getRev()->gitRev()
                         : ref.compare(0, 5, "refs/") == 0
                         ? ref
                         : ref == "HEAD"
diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh
index c38cd27eb..4985c7764 100644
--- a/tests/functional/fetchGit.sh
+++ b/tests/functional/fetchGit.sh
@@ -51,9 +51,7 @@ git -C $repo add differentbranch
 git -C $repo commit -m 'Test2'
 git -C $repo checkout master
 devrev=$(git -C $repo rev-parse devtest)
-out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; }" 2>&1) || status=$?
-[[ $status == 1 ]]
-[[ $out =~ 'Cannot find Git revision' ]]
+nix eval --impure --raw --expr "builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; }"
 
 [[ $(nix eval --raw --expr "builtins.readFile (builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; allRefs = true; } + \"/differentbranch\")") = 'different file' ]]
 

From 071f14a0bb25ffa8e5aaf8ad37031d205f49ef7d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 15 Nov 2023 17:17:07 +0100
Subject: [PATCH 055/421] Don't do shallow fetches over ssh

---
 src/libfetchers/git-utils.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index f554dcc5f..19eae0e1d 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -381,7 +381,9 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         };
 
         git_fetch_options opts = GIT_FETCH_OPTIONS_INIT;
-        opts.depth = shallow ? 1 : GIT_FETCH_DEPTH_FULL;
+        // FIXME: for some reason, shallow fetching over ssh barfs
+        // with "could not read from remote repository".
+        opts.depth = shallow && parseURL(url).scheme != "ssh" ? 1 : GIT_FETCH_DEPTH_FULL;
         opts.callbacks.payload = &act;
         opts.callbacks.sideband_progress = sidebandProgressCallback;
         opts.callbacks.transfer_progress = transferProgressCallback;

From a0162d5732b23e7fdc1f65df28826611e3a424e5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Nov 2023 18:54:36 +0100
Subject: [PATCH 056/421] Improve SourceAccessor path display
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Backported from lazy-trees. This allows SourceAccessors to show the
origin of the accessor. E.g. we now get

  copying '«git+https://github.com/blender/blender.git?ref=refs/heads/main&rev=4edc1389337dd3679ff66969c332d2aff52e1992»/' to the store

instead of

  copying '/' to the store
---
 src/libexpr/eval.cc                  | 3 +++
 src/libfetchers/fs-input-accessor.cc | 1 +
 src/libfetchers/git.cc               | 2 ++
 src/libutil/posix-source-accessor.hh | 2 +-
 src/libutil/source-accessor.cc       | 9 ++++++++-
 src/libutil/source-accessor.hh       | 4 ++++
 6 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 46a49c891..bf6b6f8c1 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -532,6 +532,9 @@ EvalState::EvalState(
     , baseEnv(allocEnv(128))
     , staticBaseEnv{std::make_shared<StaticEnv>(false, nullptr)}
 {
+    corepkgsFS->setPathDisplay("<nix", ">");
+    internalFS->setPathDisplay("«nix-internal»", "");
+
     countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0";
 
     assert(gcInitialised);
diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc
index 81be64482..2efee932d 100644
--- a/src/libfetchers/fs-input-accessor.cc
+++ b/src/libfetchers/fs-input-accessor.cc
@@ -18,6 +18,7 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor
         , allowedPaths(std::move(allowedPaths))
         , makeNotAllowedError(std::move(makeNotAllowedError))
     {
+        displayPrefix = root.isRoot() ? "" : root.abs();
     }
 
     void readFile(
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 7208a0b6d..6bca87304 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -582,6 +582,8 @@ struct GitInputScheme : InputScheme
 
         auto accessor = repo->getAccessor(rev);
 
+        accessor->setPathDisplay("«" + input.to_string() + "»");
+
         /* If the repo has submodules, fetch them and return a mounted
            input accessor consisting of the accessor for the top-level
            repo and the accessors for the submodules. */
diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh
index cf087d26e..a45d96bf8 100644
--- a/src/libutil/posix-source-accessor.hh
+++ b/src/libutil/posix-source-accessor.hh
@@ -7,7 +7,7 @@ namespace nix {
 /**
  * A source accessor that uses the Unix filesystem.
  */
-struct PosixSourceAccessor : SourceAccessor
+struct PosixSourceAccessor : virtual SourceAccessor
 {
     /**
      * The most recent mtime seen by lstat(). This is a hack to
diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc
index e2114e18f..7813433a7 100644
--- a/src/libutil/source-accessor.cc
+++ b/src/libutil/source-accessor.cc
@@ -7,6 +7,7 @@ static std::atomic<size_t> nextNumber{0};
 
 SourceAccessor::SourceAccessor()
     : number(++nextNumber)
+    , displayPrefix{"«unknown»"}
 {
 }
 
@@ -55,9 +56,15 @@ SourceAccessor::Stat SourceAccessor::lstat(const CanonPath & path)
         throw Error("path '%s' does not exist", showPath(path));
 }
 
+void SourceAccessor::setPathDisplay(std::string displayPrefix, std::string displaySuffix)
+{
+    this->displayPrefix = std::move(displayPrefix);
+    this->displaySuffix = std::move(displaySuffix);
+}
+
 std::string SourceAccessor::showPath(const CanonPath & path)
 {
-    return path.abs();
+    return displayPrefix + path.abs() + displaySuffix;
 }
 
 }
diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh
index 1a4e80361..264caab16 100644
--- a/src/libutil/source-accessor.hh
+++ b/src/libutil/source-accessor.hh
@@ -17,6 +17,8 @@ struct SourceAccessor
 {
     const size_t number;
 
+    std::string displayPrefix, displaySuffix;
+
     SourceAccessor();
 
     virtual ~SourceAccessor()
@@ -117,6 +119,8 @@ struct SourceAccessor
         return number < x.number;
     }
 
+    void setPathDisplay(std::string displayPrefix, std::string displaySuffix = "");
+
     virtual std::string showPath(const CanonPath & path);
 };
 

From 99d5204baaef211234d50f20610fa43d304888ce Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 20 Nov 2023 20:04:37 +0100
Subject: [PATCH 057/421] Persistently cache InputAccessor::fetchToStore()

This avoids repeated copying of the same source tree between Nix
invocations. It requires the accessor to have a "fingerprint" (e.g. a
Git revision) that uniquely determines its contents.
---
 src/libfetchers/fetchers.cc       |  5 +++++
 src/libfetchers/fetchers.hh       |  9 +++++++++
 src/libfetchers/git.cc            | 14 +++++++++++++-
 src/libfetchers/github.cc         |  8 ++++++++
 src/libfetchers/input-accessor.cc | 30 ++++++++++++++++++++++++++++++
 src/libfetchers/input-accessor.hh |  2 ++
 src/libfetchers/mercurial.cc      |  8 ++++++++
 src/libstore/content-address.hh   |  4 ++--
 8 files changed, 77 insertions(+), 3 deletions(-)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index c2513e076..60208619e 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -108,6 +108,11 @@ Input Input::fromAttrs(Attrs && attrs)
     return std::move(*res);
 }
 
+std::optional<std::string> Input::getFingerprint(ref<Store> store) const
+{
+    return scheme ? scheme->getFingerprint(store, *this) : std::nullopt;
+}
+
 ParsedURL Input::toURL() const
 {
     if (!scheme)
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index ce5aa4c69..5f3254b6d 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -113,6 +113,12 @@ public:
     std::optional<Hash> getRev() const;
     std::optional<uint64_t> getRevCount() const;
     std::optional<time_t> getLastModified() const;
+
+    /**
+     * For locked inputs, return a string that uniquely specifies the
+     * content of the input (typically a commit hash or content hash).
+     */
+    std::optional<std::string> getFingerprint(ref<Store> store) const;
 };
 
 
@@ -180,6 +186,9 @@ struct InputScheme
 
     virtual bool isDirect(const Input & input) const
     { return true; }
+
+    virtual std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const
+    { return std::nullopt; }
 };
 
 void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 7208a0b6d..6b461499b 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -700,10 +700,22 @@ struct GitInputScheme : InputScheme
 
         auto repoInfo = getRepoInfo(input);
 
-        return
+        auto [accessor, final] =
             input.getRef() || input.getRev() || !repoInfo.isLocal
             ? getAccessorFromCommit(store, repoInfo, std::move(input))
             : getAccessorFromWorkdir(store, repoInfo, std::move(input));
+
+        accessor->fingerprint = final.getFingerprint(store);
+
+        return {accessor, std::move(final)};
+    }
+
+    std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
+    {
+        if (auto rev = input.getRev())
+            return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : "");
+        else
+            return std::nullopt;
     }
 };
 
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 6c9b29721..661ad4884 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -229,6 +229,14 @@ struct GitArchiveInputScheme : InputScheme
     {
         return Xp::Flakes;
     }
+
+    std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
+    {
+        if (auto rev = input.getRev())
+            return rev->gitRev();
+        else
+            return std::nullopt;
+    }
 };
 
 struct GitHubInputScheme : GitArchiveInputScheme
diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc
index d1d450cf7..53502c621 100644
--- a/src/libfetchers/input-accessor.cc
+++ b/src/libfetchers/input-accessor.cc
@@ -1,5 +1,6 @@
 #include "input-accessor.hh"
 #include "store-api.hh"
+#include "cache.hh"
 
 namespace nix {
 
@@ -11,6 +12,30 @@ StorePath InputAccessor::fetchToStore(
     PathFilter * filter,
     RepairFlag repair)
 {
+    // FIXME: add an optimisation for the case where the accessor is
+    // an FSInputAccessor pointing to a store path.
+
+    std::optional<fetchers::Attrs> cacheKey;
+
+    if (!filter && fingerprint) {
+        cacheKey = fetchers::Attrs{
+            {"_what", "fetchToStore"},
+            {"store", store->storeDir},
+            {"name", std::string(name)},
+            {"fingerprint", *fingerprint},
+            {"method", (uint8_t) method},
+            {"path", path.abs()}
+        };
+        if (auto res = fetchers::getCache()->lookup(*cacheKey)) {
+            StorePath storePath(fetchers::getStrAttr(*res, "storePath"));
+            if (store->isValidPath(storePath)) {
+                debug("store path cache hit for '%s'", showPath(path));
+                return storePath;
+            }
+        }
+    } else
+        debug("source path '%s' is uncacheable", showPath(path));
+
     Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path)));
 
     auto source = sinkToSource([&](Sink & sink) {
@@ -25,6 +50,11 @@ StorePath InputAccessor::fetchToStore(
         ? store->computeStorePathFromDump(*source, name, method, htSHA256).first
         : store->addToStoreFromDump(*source, name, method, htSHA256, repair);
 
+    if (cacheKey)
+        fetchers::getCache()->upsert(
+            *cacheKey,
+            fetchers::Attrs{{"storePath", std::string(storePath.to_string())}});
+
     return storePath;
 }
 
diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh
index 9c688a234..26d17f064 100644
--- a/src/libfetchers/input-accessor.hh
+++ b/src/libfetchers/input-accessor.hh
@@ -18,6 +18,8 @@ class Store;
 
 struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this<InputAccessor>
 {
+    std::optional<std::string> fingerprint;
+
     /**
      * Return the maximum last-modified time of the files in this
      * tree, if available.
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index 9244acf39..aa991a75d 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -339,6 +339,14 @@ struct MercurialInputScheme : InputScheme
 
         return makeResult(infoAttrs, std::move(storePath));
     }
+
+    std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
+    {
+        if (auto rev = input.getRev())
+            return rev->gitRev();
+        else
+            return std::nullopt;
+    }
 };
 
 static auto rMercurialInputScheme = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh
index c4d619bdc..bdb558907 100644
--- a/src/libstore/content-address.hh
+++ b/src/libstore/content-address.hh
@@ -39,12 +39,12 @@ enum struct FileIngestionMethod : uint8_t {
     /**
      * Flat-file hashing. Directly ingest the contents of a single file
      */
-    Flat = false,
+    Flat = 0,
     /**
      * Recursive (or NAR) hashing. Serializes the file-system object in Nix
      * Archive format and ingest that
      */
-    Recursive = true
+    Recursive = 1
 };
 
 /**

From 64827360be35a3d16e818aa9d8426ca40b2c4dc2 Mon Sep 17 00:00:00 2001
From: Felix Uhl <github@mail.felix-uhl.de>
Date: Tue, 21 Nov 2023 14:49:48 +0100
Subject: [PATCH 058/421] Fix "unbound variable" errors in bash

Fixes #9414
---
 scripts/nix-profile-daemon.sh.in | 2 +-
 scripts/nix-profile.sh.in        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in
index c63db4648..d256b24ed 100644
--- a/scripts/nix-profile-daemon.sh.in
+++ b/scripts/nix-profile-daemon.sh.in
@@ -31,7 +31,7 @@ fi
 export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK"
 
 # Populate bash completions, .desktop files, etc
-if [ -z "$XDG_DATA_DIRS" ]; then
+if [ -z "${XDG_DATA_DIRS-}" ]; then
     # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
     export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
 else
diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in
index 56e070ae1..44bc96e89 100644
--- a/scripts/nix-profile.sh.in
+++ b/scripts/nix-profile.sh.in
@@ -33,7 +33,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
     export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK"
 
     # Populate bash completions, .desktop files, etc
-    if [ -z "$XDG_DATA_DIRS" ]; then
+    if [ -z "${XDG_DATA_DIRS-}" ]; then
         # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default
         export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share"
     else

From 4d8decbd135a78389c463fbb4c844f1bf22aed69 Mon Sep 17 00:00:00 2001
From: Lin Jian <me@linj.tech>
Date: Tue, 21 Nov 2023 15:29:36 +0800
Subject: [PATCH 059/421] doc: fix number of template attributes

This number is not updated when welcomeText is added[1][2].

[1]: f3a2940e70dea2c35dcae3fca019e94bf8758b4d
[2]: https://github.com/NixOS/nix/pull/6103
---
 src/nix/flake-init.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/flake-init.md b/src/nix/flake-init.md
index fc1f4f805..ea274bf29 100644
--- a/src/nix/flake-init.md
+++ b/src/nix/flake-init.md
@@ -30,7 +30,7 @@ using `-t`.
 # Template definitions
 
 A flake can declare templates through its `templates` output
-attribute. A template has two attributes:
+attribute. A template has the following attributes:
 
 * `description`: A one-line description of the template, in CommonMark
   syntax.

From f880469173061a07f0b2a24734932c5a9ad633c6 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 19 Nov 2023 10:17:57 -0500
Subject: [PATCH 060/421] Put `canonicaliseTimestampAndPermissions` in its own
 header/file

It is not inherently tied to `LocalStore`, it could probably even go in
`libnixutil`. Functions not attached to `LocalStore` should not be
declared in `local-store.hh`.

I am moving it to facilitate experimenting for #9344. If
canonicalisation should be done client-side in client-side builds, there
wouldn't be a `LocalStore` at all so having to include that header to
get this freestanding function is cumbersome and wrong.

Perhaps canonicalisation should still be done server-side for security
reasons --- I don't mean to make that judgement call now --- but even if
so, this freestanding function still isn't connected to `LocalStore` so
while less urgent it is still better to move out of this header.
---
 src/libstore/build/local-derivation-goal.cc |   1 +
 src/libstore/local-store.cc                 | 159 +-----------------
 src/libstore/local-store.hh                 |  34 ----
 src/libstore/optimise-store.cc              |   1 +
 src/libstore/posix-fs-canonicalise.cc       | 169 ++++++++++++++++++++
 src/libstore/posix-fs-canonicalise.hh       |  45 ++++++
 src/nix-store/nix-store.cc                  |   1 +
 7 files changed, 218 insertions(+), 192 deletions(-)
 create mode 100644 src/libstore/posix-fs-canonicalise.cc
 create mode 100644 src/libstore/posix-fs-canonicalise.hh

diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index a9f930773..198402ff7 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -19,6 +19,7 @@
 #include "namespaces.hh"
 #include "child.hh"
 #include "unix-domain-socket.hh"
+#include "posix-fs-canonicalise.hh"
 
 #include <regex>
 #include <queue>
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 2a3582ad8..4ff75f528 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -11,6 +11,7 @@
 #include "finally.hh"
 #include "compression.hh"
 #include "signals.hh"
+#include "posix-fs-canonicalise.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -581,164 +582,6 @@ void LocalStore::makeStoreWritable()
 }
 
 
-const time_t mtimeStore = 1; /* 1 second into the epoch */
-
-
-static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st)
-{
-    if (!S_ISLNK(st.st_mode)) {
-
-        /* Mask out all type related bits. */
-        mode_t mode = st.st_mode & ~S_IFMT;
-
-        if (mode != 0444 && mode != 0555) {
-            mode = (st.st_mode & S_IFMT)
-                 | 0444
-                 | (st.st_mode & S_IXUSR ? 0111 : 0);
-            if (chmod(path.c_str(), mode) == -1)
-                throw SysError("changing mode of '%1%' to %2$o", path, mode);
-        }
-
-    }
-
-    if (st.st_mtime != mtimeStore) {
-        struct timeval times[2];
-        times[0].tv_sec = st.st_atime;
-        times[0].tv_usec = 0;
-        times[1].tv_sec = mtimeStore;
-        times[1].tv_usec = 0;
-#if HAVE_LUTIMES
-        if (lutimes(path.c_str(), times) == -1)
-            if (errno != ENOSYS ||
-                (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1))
-#else
-        if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)
-#endif
-            throw SysError("changing modification time of '%1%'", path);
-    }
-}
-
-
-void canonicaliseTimestampAndPermissions(const Path & path)
-{
-    canonicaliseTimestampAndPermissions(path, lstat(path));
-}
-
-
-static void canonicalisePathMetaData_(
-    const Path & path,
-    std::optional<std::pair<uid_t, uid_t>> uidRange,
-    InodesSeen & inodesSeen)
-{
-    checkInterrupt();
-
-#if __APPLE__
-    /* Remove flags, in particular UF_IMMUTABLE which would prevent
-       the file from being garbage-collected. FIXME: Use
-       setattrlist() to remove other attributes as well. */
-    if (lchflags(path.c_str(), 0)) {
-        if (errno != ENOTSUP)
-            throw SysError("clearing flags of path '%1%'", path);
-    }
-#endif
-
-    auto st = lstat(path);
-
-    /* Really make sure that the path is of a supported type. */
-    if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
-        throw Error("file '%1%' has an unsupported type", path);
-
-#if __linux__
-    /* Remove extended attributes / ACLs. */
-    ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0);
-
-    if (eaSize < 0) {
-        if (errno != ENOTSUP && errno != ENODATA)
-            throw SysError("querying extended attributes of '%s'", path);
-    } else if (eaSize > 0) {
-        std::vector<char> eaBuf(eaSize);
-
-        if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0)
-            throw SysError("querying extended attributes of '%s'", path);
-
-        for (auto & eaName: tokenizeString<Strings>(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) {
-            if (settings.ignoredAcls.get().count(eaName)) continue;
-            if (lremovexattr(path.c_str(), eaName.c_str()) == -1)
-                throw SysError("removing extended attribute '%s' from '%s'", eaName, path);
-        }
-     }
-#endif
-
-    /* Fail if the file is not owned by the build user.  This prevents
-       us from messing up the ownership/permissions of files
-       hard-linked into the output (e.g. "ln /etc/shadow $out/foo").
-       However, ignore files that we chown'ed ourselves previously to
-       ensure that we don't fail on hard links within the same build
-       (i.e. "touch $out/foo; ln $out/foo $out/bar"). */
-    if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) {
-        if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino)))
-            throw BuildError("invalid ownership on file '%1%'", path);
-        mode_t mode = st.st_mode & ~S_IFMT;
-        assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));
-        return;
-    }
-
-    inodesSeen.insert(Inode(st.st_dev, st.st_ino));
-
-    canonicaliseTimestampAndPermissions(path, st);
-
-    /* Change ownership to the current uid.  If it's a symlink, use
-       lchown if available, otherwise don't bother.  Wrong ownership
-       of a symlink doesn't matter, since the owning user can't change
-       the symlink and can't delete it because the directory is not
-       writable.  The only exception is top-level paths in the Nix
-       store (since that directory is group-writable for the Nix build
-       users group); we check for this case below. */
-    if (st.st_uid != geteuid()) {
-#if HAVE_LCHOWN
-        if (lchown(path.c_str(), geteuid(), getegid()) == -1)
-#else
-        if (!S_ISLNK(st.st_mode) &&
-            chown(path.c_str(), geteuid(), getegid()) == -1)
-#endif
-            throw SysError("changing owner of '%1%' to %2%",
-                path, geteuid());
-    }
-
-    if (S_ISDIR(st.st_mode)) {
-        DirEntries entries = readDirectory(path);
-        for (auto & i : entries)
-            canonicalisePathMetaData_(path + "/" + i.name, uidRange, inodesSeen);
-    }
-}
-
-
-void canonicalisePathMetaData(
-    const Path & path,
-    std::optional<std::pair<uid_t, uid_t>> uidRange,
-    InodesSeen & inodesSeen)
-{
-    canonicalisePathMetaData_(path, uidRange, inodesSeen);
-
-    /* On platforms that don't have lchown(), the top-level path can't
-       be a symlink, since we can't change its ownership. */
-    auto st = lstat(path);
-
-    if (st.st_uid != geteuid()) {
-        assert(S_ISLNK(st.st_mode));
-        throw Error("wrong ownership of top-level store path '%1%'", path);
-    }
-}
-
-
-void canonicalisePathMetaData(const Path & path,
-    std::optional<std::pair<uid_t, uid_t>> uidRange)
-{
-    InodesSeen inodesSeen;
-    canonicalisePathMetaData(path, uidRange, inodesSeen);
-}
-
-
 void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs)
 {
     experimentalFeatureSettings.require(Xp::CaDerivations);
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 6d589bee5..8f0ffd2a2 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -371,38 +371,4 @@ private:
     friend struct DerivationGoal;
 };
 
-
-typedef std::pair<dev_t, ino_t> Inode;
-typedef std::set<Inode> InodesSeen;
-
-
-/**
- * "Fix", or canonicalise, the meta-data of the files in a store path
- * after it has been built.  In particular:
- *
- * - the last modification date on each file is set to 1 (i.e.,
- *   00:00:01 1/1/1970 UTC)
- *
- * - the permissions are set of 444 or 555 (i.e., read-only with or
- *   without execute permission; setuid bits etc. are cleared)
- *
- * - the owner and group are set to the Nix user and group, if we're
- *   running as root.
- *
- * If uidRange is not empty, this function will throw an error if it
- * encounters files owned by a user outside of the closed interval
- * [uidRange->first, uidRange->second].
- */
-void canonicalisePathMetaData(
-    const Path & path,
-    std::optional<std::pair<uid_t, uid_t>> uidRange,
-    InodesSeen & inodesSeen);
-void canonicalisePathMetaData(
-    const Path & path,
-    std::optional<std::pair<uid_t, uid_t>> uidRange);
-
-void canonicaliseTimestampAndPermissions(const Path & path);
-
-MakeError(PathInUse, Error);
-
 }
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index a4ac413b3..0fa977545 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -1,6 +1,7 @@
 #include "local-store.hh"
 #include "globals.hh"
 #include "signals.hh"
+#include "posix-fs-canonicalise.hh"
 
 #include <cstdlib>
 #include <cstring>
diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc
new file mode 100644
index 000000000..cc3ab0b74
--- /dev/null
+++ b/src/libstore/posix-fs-canonicalise.cc
@@ -0,0 +1,169 @@
+#include <sys/xattr.h>
+
+#include "posix-fs-canonicalise.hh"
+#include "file-system.hh"
+#include "signals.hh"
+#include "util.hh"
+#include "globals.hh"
+#include "store-api.hh"
+
+namespace nix {
+
+const time_t mtimeStore = 1; /* 1 second into the epoch */
+
+
+static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st)
+{
+    if (!S_ISLNK(st.st_mode)) {
+
+        /* Mask out all type related bits. */
+        mode_t mode = st.st_mode & ~S_IFMT;
+
+        if (mode != 0444 && mode != 0555) {
+            mode = (st.st_mode & S_IFMT)
+                 | 0444
+                 | (st.st_mode & S_IXUSR ? 0111 : 0);
+            if (chmod(path.c_str(), mode) == -1)
+                throw SysError("changing mode of '%1%' to %2$o", path, mode);
+        }
+
+    }
+
+    if (st.st_mtime != mtimeStore) {
+        struct timeval times[2];
+        times[0].tv_sec = st.st_atime;
+        times[0].tv_usec = 0;
+        times[1].tv_sec = mtimeStore;
+        times[1].tv_usec = 0;
+#if HAVE_LUTIMES
+        if (lutimes(path.c_str(), times) == -1)
+            if (errno != ENOSYS ||
+                (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1))
+#else
+        if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)
+#endif
+            throw SysError("changing modification time of '%1%'", path);
+    }
+}
+
+
+void canonicaliseTimestampAndPermissions(const Path & path)
+{
+    canonicaliseTimestampAndPermissions(path, lstat(path));
+}
+
+
+static void canonicalisePathMetaData_(
+    const Path & path,
+    std::optional<std::pair<uid_t, uid_t>> uidRange,
+    InodesSeen & inodesSeen)
+{
+    checkInterrupt();
+
+#if __APPLE__
+    /* Remove flags, in particular UF_IMMUTABLE which would prevent
+       the file from being garbage-collected. FIXME: Use
+       setattrlist() to remove other attributes as well. */
+    if (lchflags(path.c_str(), 0)) {
+        if (errno != ENOTSUP)
+            throw SysError("clearing flags of path '%1%'", path);
+    }
+#endif
+
+    auto st = lstat(path);
+
+    /* Really make sure that the path is of a supported type. */
+    if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
+        throw Error("file '%1%' has an unsupported type", path);
+
+#if __linux__
+    /* Remove extended attributes / ACLs. */
+    ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0);
+
+    if (eaSize < 0) {
+        if (errno != ENOTSUP && errno != ENODATA)
+            throw SysError("querying extended attributes of '%s'", path);
+    } else if (eaSize > 0) {
+        std::vector<char> eaBuf(eaSize);
+
+        if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0)
+            throw SysError("querying extended attributes of '%s'", path);
+
+        for (auto & eaName: tokenizeString<Strings>(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) {
+            if (settings.ignoredAcls.get().count(eaName)) continue;
+            if (lremovexattr(path.c_str(), eaName.c_str()) == -1)
+                throw SysError("removing extended attribute '%s' from '%s'", eaName, path);
+        }
+     }
+#endif
+
+    /* Fail if the file is not owned by the build user.  This prevents
+       us from messing up the ownership/permissions of files
+       hard-linked into the output (e.g. "ln /etc/shadow $out/foo").
+       However, ignore files that we chown'ed ourselves previously to
+       ensure that we don't fail on hard links within the same build
+       (i.e. "touch $out/foo; ln $out/foo $out/bar"). */
+    if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) {
+        if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino)))
+            throw BuildError("invalid ownership on file '%1%'", path);
+        mode_t mode = st.st_mode & ~S_IFMT;
+        assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));
+        return;
+    }
+
+    inodesSeen.insert(Inode(st.st_dev, st.st_ino));
+
+    canonicaliseTimestampAndPermissions(path, st);
+
+    /* Change ownership to the current uid.  If it's a symlink, use
+       lchown if available, otherwise don't bother.  Wrong ownership
+       of a symlink doesn't matter, since the owning user can't change
+       the symlink and can't delete it because the directory is not
+       writable.  The only exception is top-level paths in the Nix
+       store (since that directory is group-writable for the Nix build
+       users group); we check for this case below. */
+    if (st.st_uid != geteuid()) {
+#if HAVE_LCHOWN
+        if (lchown(path.c_str(), geteuid(), getegid()) == -1)
+#else
+        if (!S_ISLNK(st.st_mode) &&
+            chown(path.c_str(), geteuid(), getegid()) == -1)
+#endif
+            throw SysError("changing owner of '%1%' to %2%",
+                path, geteuid());
+    }
+
+    if (S_ISDIR(st.st_mode)) {
+        DirEntries entries = readDirectory(path);
+        for (auto & i : entries)
+            canonicalisePathMetaData_(path + "/" + i.name, uidRange, inodesSeen);
+    }
+}
+
+
+void canonicalisePathMetaData(
+    const Path & path,
+    std::optional<std::pair<uid_t, uid_t>> uidRange,
+    InodesSeen & inodesSeen)
+{
+    canonicalisePathMetaData_(path, uidRange, inodesSeen);
+
+    /* On platforms that don't have lchown(), the top-level path can't
+       be a symlink, since we can't change its ownership. */
+    auto st = lstat(path);
+
+    if (st.st_uid != geteuid()) {
+        assert(S_ISLNK(st.st_mode));
+        throw Error("wrong ownership of top-level store path '%1%'", path);
+    }
+}
+
+
+void canonicalisePathMetaData(const Path & path,
+    std::optional<std::pair<uid_t, uid_t>> uidRange)
+{
+    InodesSeen inodesSeen;
+    canonicalisePathMetaData(path, uidRange, inodesSeen);
+}
+
+}
diff --git a/src/libstore/posix-fs-canonicalise.hh b/src/libstore/posix-fs-canonicalise.hh
new file mode 100644
index 000000000..35644af12
--- /dev/null
+++ b/src/libstore/posix-fs-canonicalise.hh
@@ -0,0 +1,45 @@
+#pragma once
+///@file
+
+#include <sys/stat.h>
+#include <sys/time.h>
+
+#include "types.hh"
+#include "error.hh"
+
+namespace nix {
+
+typedef std::pair<dev_t, ino_t> Inode;
+typedef std::set<Inode> InodesSeen;
+
+
+/**
+ * "Fix", or canonicalise, the meta-data of the files in a store path
+ * after it has been built.  In particular:
+ *
+ * - the last modification date on each file is set to 1 (i.e.,
+ *   00:00:01 1/1/1970 UTC)
+ *
+ * - the permissions are set of 444 or 555 (i.e., read-only with or
+ *   without execute permission; setuid bits etc. are cleared)
+ *
+ * - the owner and group are set to the Nix user and group, if we're
+ *   running as root.
+ *
+ * If uidRange is not empty, this function will throw an error if it
+ * encounters files owned by a user outside of the closed interval
+ * [uidRange->first, uidRange->second].
+ */
+void canonicalisePathMetaData(
+    const Path & path,
+    std::optional<std::pair<uid_t, uid_t>> uidRange,
+    InodesSeen & inodesSeen);
+void canonicalisePathMetaData(
+    const Path & path,
+    std::optional<std::pair<uid_t, uid_t>> uidRange);
+
+void canonicaliseTimestampAndPermissions(const Path & path);
+
+MakeError(PathInUse, Error);
+
+}
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 123283dfe..25f0107bc 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -14,6 +14,7 @@
 #include "graphml.hh"
 #include "legacy.hh"
 #include "path-with-outputs.hh"
+#include "posix-fs-canonicalise.hh"
 
 #include <iostream>
 #include <algorithm>

From 949f5841f8a8611d0f49793bd8c4963462d62e3a Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 21 Nov 2023 13:12:55 -0500
Subject: [PATCH 061/421] Add the `MountedSSHStore` experimental feature

It will be implemented in the subsequent commits of this PR.
---
 src/libutil/experimental-features.cc | 7 +++++++
 src/libutil/experimental-features.hh | 1 +
 2 files changed, 8 insertions(+)

diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index ac4d189e1..2418e3f4c 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -262,6 +262,13 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
             Allow the use of the [impure-env](@docroot@/command-ref/conf-file.md#conf-impure-env) setting.
         )",
     },
+    {
+        .tag = Xp::MountedSSHStore,
+        .name = "mounted-ssh-store",
+        .description = R"(
+            Allow the use of the [`mounted SSH store`](@docroot@/command-ref/new-cli/nix3-help-stores.html#experimental-ssh-store-with-filesytem-mounted).
+        )",
+    },
     {
         .tag = Xp::VerifiedFetches,
         .name = "verified-fetches",
diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh
index c355b8081..eae4fa9b8 100644
--- a/src/libutil/experimental-features.hh
+++ b/src/libutil/experimental-features.hh
@@ -34,6 +34,7 @@ enum struct ExperimentalFeature
     ParseTomlTimestamps,
     ReadOnlyLocalStore,
     ConfigurableImpureEnv,
+    MountedSSHStore,
     VerifiedFetches,
 };
 

From 9796ebd7ef8c9a23ee8128273d925acae00e43b0 Mon Sep 17 00:00:00 2001
From: Matej Urbas <matej.urbas@gmail.com>
Date: Mon, 17 Apr 2023 12:08:42 -0400
Subject: [PATCH 062/421] Add `--process-ops` flag to `nix-daemon`

---
 src/nix/daemon.cc | 33 +++++++++++++++++++++++++++++----
 1 file changed, 29 insertions(+), 4 deletions(-)

diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc
index 373dedf7c..4dada8e0e 100644
--- a/src/nix/daemon.cc
+++ b/src/nix/daemon.cc
@@ -443,16 +443,23 @@ static void processStdioConnection(ref<Store> store, TrustedFlag trustClient)
  *
  * @param forceTrustClientOpt See `daemonLoop()` and the parameter with
  * the same name over there for details.
+ *
+ * @param procesOps Whether to force processing ops even if the next
+ * store also is a remote store and could process it directly.
  */
-static void runDaemon(bool stdio, std::optional<TrustedFlag> forceTrustClientOpt)
+static void runDaemon(bool stdio, std::optional<TrustedFlag> forceTrustClientOpt, bool processOps)
 {
     if (stdio) {
         auto store = openUncachedStore();
 
+        std::shared_ptr<RemoteStore> remoteStore;
+
         // If --force-untrusted is passed, we cannot forward the connection and
         // must process it ourselves (before delegating to the next store) to
         // force untrusting the client.
-        if (auto remoteStore = store.dynamic_pointer_cast<RemoteStore>(); remoteStore && (!forceTrustClientOpt || *forceTrustClientOpt != NotTrusted))
+        processOps |= !forceTrustClientOpt || *forceTrustClientOpt != NotTrusted;
+
+        if (!processOps && (remoteStore = store.dynamic_pointer_cast<RemoteStore>()))
             forwardStdioConnection(*remoteStore);
         else
             // `Trusted` is passed in the auto (no override case) because we
@@ -468,6 +475,7 @@ static int main_nix_daemon(int argc, char * * argv)
     {
         auto stdio = false;
         std::optional<TrustedFlag> isTrustedOpt = std::nullopt;
+        auto processOps = false;
 
         parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
             if (*arg == "--daemon")
@@ -487,11 +495,14 @@ static int main_nix_daemon(int argc, char * * argv)
             } else if (*arg == "--default-trust") {
                 experimentalFeatureSettings.require(Xp::DaemonTrustOverride);
                 isTrustedOpt = std::nullopt;
+            } else if (*arg == "--process-ops") {
+                experimentalFeatureSettings.require(Xp::MountedSSHStore);
+                processOps = true;
             } else return false;
             return true;
         });
 
-        runDaemon(stdio, isTrustedOpt);
+        runDaemon(stdio, isTrustedOpt, processOps);
 
         return 0;
     }
@@ -503,6 +514,7 @@ struct CmdDaemon : StoreCommand
 {
     bool stdio = false;
     std::optional<TrustedFlag> isTrustedOpt = std::nullopt;
+    bool processOps = false;
 
     CmdDaemon()
     {
@@ -538,6 +550,19 @@ struct CmdDaemon : StoreCommand
             }},
             .experimentalFeature = Xp::DaemonTrustOverride,
         });
+
+        addFlag({
+            .longName = "process-ops",
+            .description = R"(
+              Forces the daemon to process received commands itself rather than forwarding the commands straight to the remote store.
+
+              This is useful for the `mounted-ssh://` store where some actions need to be performed on the remote end but as connected user, and not as the user of the underlying daemon on the remote end.
+            )",
+            .handler = {[&]() {
+                processOps = true;
+            }},
+            .experimentalFeature = Xp::MountedSSHStore,
+        });
     }
 
     std::string description() override
@@ -556,7 +581,7 @@ struct CmdDaemon : StoreCommand
 
     void run(ref<Store> store) override
     {
-        runDaemon(stdio, isTrustedOpt);
+        runDaemon(stdio, isTrustedOpt, processOps);
     }
 };
 

From 226b0f3956ef83ec51b6556d8f27e13a966b4ebf Mon Sep 17 00:00:00 2001
From: Matej Urbas <matej.urbas@gmail.com>
Date: Mon, 17 Apr 2023 12:04:18 -0400
Subject: [PATCH 063/421] Extend the worker protocol with `wopAddPermRoot`

---
 src/libstore/daemon.cc          | 15 +++++++++++++++
 src/libstore/worker-protocol.hh |  3 ++-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 105d92f25..be9b0b0d3 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -657,6 +657,21 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
+    case WorkerProto::Op::AddPermRoot: {
+        if (!trusted)
+            throw Error(
+                "you are not privileged to create perm roots\n\n"
+                "hint: you can just do this client-side without special privileges, and probably want to do that instead.");
+        auto storePath = WorkerProto::Serialise<StorePath>::read(*store, rconn);
+        Path gcRoot = absPath(readString(from));
+        logger->startWork();
+        auto & localFSStore = require<LocalFSStore>(*store);
+        localFSStore.addPermRoot(storePath, gcRoot);
+        logger->stopWork();
+        to << gcRoot;
+        break;
+    }
+
     case WorkerProto::Op::AddIndirectRoot: {
         Path path = absPath(readString(from));
 
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 25d544ba7..8a26c09c5 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -9,7 +9,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION (1 << 8 | 35)
+#define PROTOCOL_VERSION (1 << 8 | 36)
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
@@ -161,6 +161,7 @@ enum struct WorkerProto::Op : uint64_t
     AddMultipleToStore = 44,
     AddBuildLog = 45,
     BuildPathsWithResults = 46,
+    AddPermRoot = 47,
 };
 
 /**

From 06b8902562089811e5724aa0b8d719f891ab73f2 Mon Sep 17 00:00:00 2001
From: Matej Urbas <matej.urbas@gmail.com>
Date: Sat, 15 Apr 2023 11:02:41 +0100
Subject: [PATCH 064/421] MountedSSHStore: stores on shared filesystems

---
 src/libstore/indirect-root-store.hh |  24 ++++++
 src/libstore/mounted-ssh-store.md   |  18 ++++
 src/libstore/ssh-store.cc           | 123 +++++++++++++++++++++++++++-
 3 files changed, 164 insertions(+), 1 deletion(-)
 create mode 100644 src/libstore/mounted-ssh-store.md

diff --git a/src/libstore/indirect-root-store.hh b/src/libstore/indirect-root-store.hh
index 59e45af45..c11679fe8 100644
--- a/src/libstore/indirect-root-store.hh
+++ b/src/libstore/indirect-root-store.hh
@@ -11,6 +11,30 @@ namespace nix {
  * reference.
  *
  * See methods for details on the operations it represents.
+ *
+ * @note
+ * To understand the purpose of this class, it might help to do some
+ * "closed-world" rather than "open-world" reasoning, and consider the
+ * problem it solved for us. This class was factored out from
+ * `LocalFSStore` in order to support the following table, which
+ * contains 4 concrete store types (non-abstract classes, exposed to the
+ * user), and how they implemented the two GC root methods:
+ *
+ * @note
+ * |                   | `addPermRoot()` | `addIndirectRoot()` |
+ * |-------------------|-----------------|---------------------|
+ * | `LocalStore`      | local           | local               |
+ * | `UDSRemoteStore`  | local           | remote              |
+ * | `SSHStore`        | doesn't have    | doesn't have        |
+ * | `MountedSSHStore` | remote          | doesn't have        |
+ *
+ * @note
+ * Note how only the local implementations of `addPermRoot()` need
+ * `addIndirectRoot()`; that is what this class enforces. Without it,
+ * and with `addPermRoot()` and `addIndirectRoot()` both `virtual`, we
+ * would accidentally be allowing for a combinatorial explosion of
+ * possible implementations many of which make no sense. Having this and
+ * that invariant enforced cuts down that space.
  */
 struct IndirectRootStore : public virtual LocalFSStore
 {
diff --git a/src/libstore/mounted-ssh-store.md b/src/libstore/mounted-ssh-store.md
new file mode 100644
index 000000000..1ebfe3081
--- /dev/null
+++ b/src/libstore/mounted-ssh-store.md
@@ -0,0 +1,18 @@
+R"(
+
+**Store URL format**: `mounted-ssh-ng://[username@]hostname`
+
+Experimental store type that allows full access to a Nix store on a remote machine,
+and additionally requires that store be mounted in the local file system.
+
+The mounting of that store is not managed by Nix, and must by managed manually.
+It could be accomplished with SSHFS or NFS, for example.
+
+The local file system is used to optimize certain operations.
+For example, rather than serializing Nix archives and sending over the Nix channel,
+we can directly access the file system data via the mount-point.
+
+The local file system is also used to make certain operations possible that wouldn't otherwise be.
+For example, persistent GC roots can be created if they reside on the same file system as the remote store:
+the remote side will create the symlinks necessary to avoid race conditions.
+)"
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 4a6aad449..d4c8ab5b2 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -3,9 +3,10 @@
 #include "local-fs-store.hh"
 #include "remote-store.hh"
 #include "remote-store-connection.hh"
-#include "remote-fs-accessor.hh"
+#include "source-accessor.hh"
 #include "archive.hh"
 #include "worker-protocol.hh"
+#include "worker-protocol-impl.hh"
 #include "pool.hh"
 #include "ssh.hh"
 
@@ -78,6 +79,8 @@ protected:
 
     std::string host;
 
+    std::vector<std::string> extraRemoteProgramArgs;
+
     SSHMaster master;
 
     void setOptions(RemoteStore::Connection & conn) override
@@ -91,6 +94,121 @@ protected:
     };
 };
 
+struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig
+{
+    using SSHStoreConfig::SSHStoreConfig;
+    using LocalFSStoreConfig::LocalFSStoreConfig;
+
+    MountedSSHStoreConfig(StringMap params)
+        : StoreConfig(params)
+        , RemoteStoreConfig(params)
+        , CommonSSHStoreConfig(params)
+        , SSHStoreConfig(params)
+        , LocalFSStoreConfig(params)
+    {
+    }
+
+    const std::string name() override { return "Experimental SSH Store with filesytem mounted"; }
+
+    std::string doc() override
+    {
+        return
+          #include "mounted-ssh-store.md"
+          ;
+    }
+
+    std::optional<ExperimentalFeature> experimentalFeature() const override
+    {
+        return ExperimentalFeature::MountedSSHStore;
+    }
+};
+
+/**
+ * The mounted ssh store assumes that filesystems on the remote host are
+ * shared with the local host. This means that the remote nix store is
+ * available locally and is therefore treated as a local filesystem
+ * store.
+ *
+ * MountedSSHStore is very similar to UDSRemoteStore --- ignoring the
+ * superficial differnce of SSH vs Unix domain sockets, they both are
+ * accessing remote stores, and they both assume the store will be
+ * mounted in the local filesystem.
+ *
+ * The difference lies in how they manage GC roots. See addPermRoot
+ * below for details.
+ */
+class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSHStore, public virtual LocalFSStore
+{
+public:
+
+    MountedSSHStore(const std::string & scheme, const std::string & host, const Params & params)
+        : StoreConfig(params)
+        , RemoteStoreConfig(params)
+        , CommonSSHStoreConfig(params)
+        , SSHStoreConfig(params)
+        , LocalFSStoreConfig(params)
+        , MountedSSHStoreConfig(params)
+        , Store(params)
+        , RemoteStore(params)
+        , SSHStore(scheme, host, params)
+        , LocalFSStore(params)
+    {
+        extraRemoteProgramArgs = {
+            "--process-ops",
+        };
+    }
+
+    static std::set<std::string> uriSchemes()
+    {
+        return {"mounted-ssh-ng"};
+    }
+
+    std::string getUri() override
+    {
+        return *uriSchemes().begin() + "://" + host;
+    }
+
+    void narFromPath(const StorePath & path, Sink & sink) override
+    {
+        return LocalFSStore::narFromPath(path, sink);
+    }
+
+    ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
+    {
+        return LocalFSStore::getFSAccessor(requireValidPath);
+    }
+
+    std::optional<std::string> getBuildLogExact(const StorePath & path) override
+    {
+        return LocalFSStore::getBuildLogExact(path);
+    }
+
+    /**
+     * This is the key difference from UDSRemoteStore: UDSRemote store
+     * has the client create the direct root, and the remote side create
+     * the indirect root.
+     *
+     * We could also do that, but the race conditions (will the remote
+     * side see the direct root the client made?) seems bigger.
+     *
+     * In addition, the remote-side will have a process associated with
+     * the authenticating user handling the connection (even if there
+     * is a system-wide daemon or similar). This process can safely make
+     * the direct and indirect roots without there being such a risk of
+     * privilege escalation / symlinks in directories owned by the
+     * originating requester that they cannot delete.
+     */
+    Path addPermRoot(const StorePath & path, const Path & gcRoot) override
+    {
+        auto conn(getConnection());
+        conn->to << WorkerProto::Op::AddPermRoot;
+        WorkerProto::write(*this, *conn, path);
+        WorkerProto::write(*this, *conn, gcRoot);
+        conn.processStderr();
+        return readString(conn->from);
+    }
+};
+
 ref<RemoteStore::Connection> SSHStore::openConnection()
 {
     auto conn = make_ref<Connection>();
@@ -98,6 +216,8 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
     std::string command = remoteProgram + " --stdio";
     if (remoteStore.get() != "")
         command += " --store " + shellEscape(remoteStore.get());
+    for (auto & arg : extraRemoteProgramArgs)
+        command += " " + shellEscape(arg);
 
     conn->sshConn = master.startCommand(command);
     conn->to = FdSink(conn->sshConn->in.get());
@@ -106,5 +226,6 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
 }
 
 static RegisterStoreImplementation<SSHStore, SSHStoreConfig> regSSHStore;
+static RegisterStoreImplementation<MountedSSHStore, MountedSSHStoreConfig> regMountedSSHStore;
 
 }

From b32b20a6d7cf3a9cf2c81a133c255e9c2ee8e308 Mon Sep 17 00:00:00 2001
From: mupdt <25388474+mupdt@users.noreply.github.com>
Date: Mon, 31 Jul 2023 09:07:43 -0400
Subject: [PATCH 065/421] release note entry for the `mounted-ssh-ng://` store

---
 doc/manual/src/release-notes/rl-next.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
index dbe2692f9..0e3d8b462 100644
--- a/doc/manual/src/release-notes/rl-next.md
+++ b/doc/manual/src/release-notes/rl-next.md
@@ -1,3 +1,6 @@
 # Release X.Y (202?-??-??)
 
-- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.
\ No newline at end of file
+- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.
+
+- Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
+  This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem.

From 03c3af1bf97354e281230a82f76d84b2db65db91 Mon Sep 17 00:00:00 2001
From: Matej Urbas <matej.urbas@gmail.com>
Date: Tue, 29 Aug 2023 17:01:09 +0100
Subject: [PATCH 066/421] mounted-ssh-ng store: integration tests

---
 .../build-remote-with-mounted-ssh-ng.sh       | 22 +++++++++++++++++++
 tests/functional/local.mk                     |  1 +
 2 files changed, 23 insertions(+)
 create mode 100644 tests/functional/build-remote-with-mounted-ssh-ng.sh

diff --git a/tests/functional/build-remote-with-mounted-ssh-ng.sh b/tests/functional/build-remote-with-mounted-ssh-ng.sh
new file mode 100644
index 000000000..443acb6ca
--- /dev/null
+++ b/tests/functional/build-remote-with-mounted-ssh-ng.sh
@@ -0,0 +1,22 @@
+source common.sh
+
+requireSandboxSupport
+[[ $busybox =~ busybox ]] || skipTest "no busybox"
+
+enableFeatures mounted-ssh-store
+
+nix build -Lvf simple.nix \
+  --arg busybox $busybox \
+  --out-link $TEST_ROOT/result-from-remote \
+  --store mounted-ssh-ng://localhost
+
+nix build -Lvf simple.nix \
+  --arg busybox $busybox \
+  --out-link $TEST_ROOT/result-from-remote-new-cli \
+  --store 'mounted-ssh-ng://localhost?remote-program=nix daemon'
+
+# This verifies that the out link was actually created and valid. The ability
+# to create out links (permanent gc roots) is the distinguishing feature of
+# the mounted-ssh-ng store.
+cat $TEST_ROOT/result-from-remote/hello | grepQuiet 'Hello World!'
+cat $TEST_ROOT/result-from-remote-new-cli/hello | grepQuiet 'Hello World!'
diff --git a/tests/functional/local.mk b/tests/functional/local.mk
index 21dabca88..8d584142a 100644
--- a/tests/functional/local.mk
+++ b/tests/functional/local.mk
@@ -69,6 +69,7 @@ nix_tests = \
   build-remote-trustless-should-pass-2.sh \
   build-remote-trustless-should-pass-3.sh \
   build-remote-trustless-should-fail-0.sh \
+  build-remote-with-mounted-ssh-ng.sh \
   nar-access.sh \
   pure-eval.sh \
   eval.sh \

From 4e790efade0c3073292ff73be44351f29badd935 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 20 Nov 2023 13:38:52 +0100
Subject: [PATCH 067/421] Use boost::container::small_vector in place of VLAs

---
 boehmgc-traceable_allocator-public.diff | 12 +++++++
 flake.nix                               |  3 ++
 src/libexpr/eval.cc                     | 13 +++++---
 src/libexpr/gc-small-vector.hh          | 42 +++++++++++++++++++++++++
 src/libexpr/primops.cc                  |  9 +++---
 5 files changed, 70 insertions(+), 9 deletions(-)
 create mode 100644 boehmgc-traceable_allocator-public.diff
 create mode 100644 src/libexpr/gc-small-vector.hh

diff --git a/boehmgc-traceable_allocator-public.diff b/boehmgc-traceable_allocator-public.diff
new file mode 100644
index 000000000..903c707a6
--- /dev/null
+++ b/boehmgc-traceable_allocator-public.diff
@@ -0,0 +1,12 @@
+diff --git a/include/gc_allocator.h b/include/gc_allocator.h
+index 597c7f13..587286be 100644
+--- a/include/gc_allocator.h
++++ b/include/gc_allocator.h
+@@ -312,6 +312,7 @@ public:
+ 
+ template<>
+ class traceable_allocator<void> {
++public:
+   typedef size_t      size_type;
+   typedef ptrdiff_t   difference_type;
+   typedef void*       pointer;
diff --git a/flake.nix b/flake.nix
index 9030a74f7..570a099ab 100644
--- a/flake.nix
+++ b/flake.nix
@@ -230,6 +230,9 @@
             }).overrideAttrs(o: {
               patches = (o.patches or []) ++ [
                 ./boehmgc-coroutine-sp-fallback.diff
+
+                # https://github.com/ivmai/bdwgc/pull/586
+                ./boehmgc-traceable_allocator-public.diff
               ];
             })
             )
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 46a49c891..90f04d40a 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -16,6 +16,7 @@
 #include "fs-input-accessor.hh"
 #include "memory-input-accessor.hh"
 #include "signals.hh"
+#include "gc-small-vector.hh"
 
 #include <algorithm>
 #include <chrono>
@@ -31,6 +32,7 @@
 
 #include <sys/resource.h>
 #include <nlohmann/json.hpp>
+#include <boost/container/small_vector.hpp>
 
 #if HAVE_BOEHMGC
 
@@ -1709,7 +1711,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 /* We have all the arguments, so call the primop with
                    the previous and new arguments. */
 
-                Value * vArgs[arity];
+                Value * vArgs[maxPrimOpArity];
                 auto n = argsDone;
                 for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left)
                     vArgs[--n] = arg->primOpApp.right;
@@ -1772,11 +1774,11 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v)
     // 4: about 60
     // 5: under 10
     // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total.
-    Value * vArgs[args.size()];
+    SmallValueVector<4> vArgs(args.size());
     for (size_t i = 0; i < args.size(); ++i)
         vArgs[i] = args[i]->maybeThunk(state, env);
 
-    state.callFunction(vFun, args.size(), vArgs, v, pos);
+    state.callFunction(vFun, args.size(), vArgs.data(), v, pos);
 }
 
 
@@ -2015,8 +2017,9 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
         return result;
     };
 
-    Value values[es->size()];
-    Value * vTmpP = values;
+    // List of returned strings. References to these Values must NOT be persisted.
+    SmallTemporaryValueVector<conservativeStackReservation> values(es->size());
+    Value * vTmpP = values.data();
 
     for (auto & [i_pos, i] : *es) {
         Value & vTmp = *vTmpP++;
diff --git a/src/libexpr/gc-small-vector.hh b/src/libexpr/gc-small-vector.hh
new file mode 100644
index 000000000..7f4f08fc7
--- /dev/null
+++ b/src/libexpr/gc-small-vector.hh
@@ -0,0 +1,42 @@
+#pragma once
+
+#include <boost/container/small_vector.hpp>
+
+#if HAVE_BOEHMGC
+
+#include <gc/gc.h>
+#include <gc/gc_cpp.h>
+#include <gc/gc_allocator.h>
+
+#endif
+
+namespace nix {
+
+struct Value;
+
+/**
+ * A GC compatible vector that may used a reserved portion of `nItems` on the stack instead of allocating on the heap.
+ */
+#if HAVE_BOEHMGC
+template <typename T, size_t nItems>
+using SmallVector = boost::container::small_vector<T, nItems, traceable_allocator<T>>;
+#else
+template <typename T, size_t nItems>
+using SmallVector = boost::container::small_vector<T, nItems>;
+#endif
+
+/**
+ * A vector of value pointers. See `SmallVector`.
+ */
+template <size_t nItems>
+using SmallValueVector = SmallVector<Value *, nItems>;
+
+/**
+ * A vector of values that must not be referenced after the vector is destroyed.
+ *
+ * See also `SmallValueVector`.
+ */
+template <size_t nItems>
+using SmallTemporaryValueVector = SmallVector<Value, nItems>;
+
+}
\ No newline at end of file
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index a8d44d8b7..54a5da817 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -4,6 +4,7 @@
 #include "eval-inline.hh"
 #include "eval.hh"
 #include "eval-settings.hh"
+#include "gc-small-vector.hh"
 #include "globals.hh"
 #include "json-to-value.hh"
 #include "names.hh"
@@ -2729,7 +2730,7 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V
     auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs"));
     state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs");
 
-    Value * res[args[1]->listSize()];
+    SmallValueVector<nonRecursiveStackReservation> res(args[1]->listSize());
     size_t found = 0;
 
     for (auto v2 : args[1]->listItems()) {
@@ -3064,8 +3065,7 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val
 
     state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter");
 
-    // FIXME: putting this on the stack is risky.
-    Value * vs[args[1]->listSize()];
+    SmallValueVector<nonRecursiveStackReservation> vs(args[1]->listSize());
     size_t k = 0;
 
     bool same = true;
@@ -3454,7 +3454,8 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args,
     state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap");
     auto nrLists = args[1]->listSize();
 
-    Value lists[nrLists];
+    // List of returned lists before concatenation. References to these Values must NOT be persisted.
+    SmallTemporaryValueVector<conservativeStackReservation> lists(nrLists);
     size_t len = 0;
 
     for (unsigned int n = 0; n < nrLists; ++n) {

From 46131567da96ffac298b9ec54016b37114b0dfd5 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 21 Nov 2023 23:19:25 -0500
Subject: [PATCH 068/421] Add missing `-lrapidcheck` fixing build with shared
 lib

https://github.com/NixOS/nixpkgs/pull/269064 makes rapidcheck be build
as a shared lib, but that broke Nix because the `-lrapidcheck` was
missing. This fixes that (and doesn't break Nix what the library is a
static archive as today).
---
 src/libexpr/tests/local.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk
index 6d2a04aaf..7689a03e0 100644
--- a/src/libexpr/tests/local.mk
+++ b/src/libexpr/tests/local.mk
@@ -20,4 +20,4 @@ libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/l
 
 libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers
 
-libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock
+libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock

From 61b76f5f34db7f863a6f22bd9083f677b339fcf6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 22 Nov 2023 11:26:12 +0100
Subject: [PATCH 069/421] Apply suggestion

Co-authored-by: John Ericson <git@JohnEricson.me>
---
 src/libfetchers/input-accessor.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc
index 53502c621..8e10cf2e2 100644
--- a/src/libfetchers/input-accessor.cc
+++ b/src/libfetchers/input-accessor.cc
@@ -27,7 +27,7 @@ StorePath InputAccessor::fetchToStore(
             {"path", path.abs()}
         };
         if (auto res = fetchers::getCache()->lookup(*cacheKey)) {
-            StorePath storePath(fetchers::getStrAttr(*res, "storePath"));
+            StorePath storePath{fetchers::getStrAttr(*res, "storePath")};
             if (store->isValidPath(storePath)) {
                 debug("store path cache hit for '%s'", showPath(path));
                 return storePath;

From 5292f364267eb74005a2e06dfe69c0d0dc8bd2a3 Mon Sep 17 00:00:00 2001
From: r-vdp <ramses@well-founded.dev>
Date: Wed, 22 Nov 2023 11:33:25 +0100
Subject: [PATCH 070/421] Fix compile warning due to unused variable binding.

We still need the check, since we don't have narinfo for locally built store paths.
---
 src/nix/path-info.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index 23198a120..080d6bbf1 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -54,7 +54,7 @@ static json pathInfoToJSON(
 
                 jsonObject["closureSize"] = getStoreObjectsTotalSize(store, closure);
 
-                if (auto * narInfo = dynamic_cast<const NarInfo *>(&*info)) {
+                if (dynamic_cast<const NarInfo *>(&*info)) {
                     uint64_t totalDownloadSize = 0;
                     for (auto & p : closure) {
                         auto depInfo = store.queryPathInfo(p);

From b1ab592f28f08da5dc7c060e5c3b19dc66dbc111 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 22 Nov 2023 11:44:02 +0100
Subject: [PATCH 071/421] Use the StorePath-based cache interface

---
 src/libfetchers/input-accessor.cc | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc
index 8e10cf2e2..85dc4609f 100644
--- a/src/libfetchers/input-accessor.cc
+++ b/src/libfetchers/input-accessor.cc
@@ -26,12 +26,9 @@ StorePath InputAccessor::fetchToStore(
             {"method", (uint8_t) method},
             {"path", path.abs()}
         };
-        if (auto res = fetchers::getCache()->lookup(*cacheKey)) {
-            StorePath storePath{fetchers::getStrAttr(*res, "storePath")};
-            if (store->isValidPath(storePath)) {
-                debug("store path cache hit for '%s'", showPath(path));
-                return storePath;
-            }
+        if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) {
+            debug("store path cache hit for '%s'", showPath(path));
+            return res->second;
         }
     } else
         debug("source path '%s' is uncacheable", showPath(path));
@@ -51,9 +48,7 @@ StorePath InputAccessor::fetchToStore(
         : store->addToStoreFromDump(*source, name, method, htSHA256, repair);
 
     if (cacheKey)
-        fetchers::getCache()->upsert(
-            *cacheKey,
-            fetchers::Attrs{{"storePath", std::string(storePath.to_string())}});
+        fetchers::getCache()->add(store, *cacheKey, {}, storePath, true);
 
     return storePath;
 }

From 2ce8c9650b3e714f28d8685e48996141cba2df2c Mon Sep 17 00:00:00 2001
From: Philipp Schuster <phip1611@gmail.com>
Date: Thu, 23 Nov 2023 22:02:20 +0100
Subject: [PATCH 072/421] doc: primops: add more info for foldl (#9254)

* doc: primops: add more info for foldl

From the existing doc it is not obvious whether the first or the
second argument is the accumulator. This is however relevant to
know, as for certain scenarios, this might change the behavior.

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 src/libexpr/primops.cc | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index a8d44d8b7..7c0561413 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -3179,9 +3179,16 @@ static RegisterPrimOp primop_foldlStrict({
     .doc = R"(
       Reduce a list by applying a binary operator, from left to right,
       e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2)
-      ...`. For example, `foldl' (x: y: x + y) 0 [1 2 3]` evaluates to 6.
-      The return value of each application of `op` is evaluated immediately,
-      even for intermediate values.
+      ...`.
+
+      For example, `foldl' (acc: elem: acc + elem) 0 [1 2 3]` evaluates
+      to `6` and `foldl' (acc: elem: { "${elem}" = elem; } // acc) {}
+      ["a" "b"]` evaluates to `{ a = "a"; b = "b"; }`.
+
+      The first argument of `op` is the accumulator wheres the second
+      argument is the current element being processed. The return value
+      of each application of `op` is evaluated immediately, even for
+      intermediate values.
     )",
     .fun = prim_foldlStrict,
 });

From 5be0e6b314c216b0b51499fc488ca08272297469 Mon Sep 17 00:00:00 2001
From: Philipp Schuster <phip1611@gmail.com>
Date: Fri, 24 Nov 2023 10:50:01 +0100
Subject: [PATCH 073/421] doc: primops: fix typo

---
 src/libexpr/primops.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 7c0561413..ba735b435 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -3185,7 +3185,7 @@ static RegisterPrimOp primop_foldlStrict({
       to `6` and `foldl' (acc: elem: { "${elem}" = elem; } // acc) {}
       ["a" "b"]` evaluates to `{ a = "a"; b = "b"; }`.
 
-      The first argument of `op` is the accumulator wheres the second
+      The first argument of `op` is the accumulator whereas the second
       argument is the current element being processed. The return value
       of each application of `op` is evaluated immediately, even for
       intermediate values.

From 6a94755b1240be654cadb463a9f528eeccf3787c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Fri, 24 Nov 2023 11:45:37 +0100
Subject: [PATCH 074/421] Allow user input in `git commit`

We occasionnally commit to git repositories (like with `nix flake update --commit-lock-file`).
This shells out to `git commit`, which might wait for user input (for a signing key passphrase for instance).
Disable the progress bar while this is running to make sure that the
user can enter it.
---
 src/libfetchers/git.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 2c5b70f53..8cd74057c 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -11,6 +11,8 @@
 #include "fs-input-accessor.hh"
 #include "mounted-input-accessor.hh"
 #include "git-utils.hh"
+#include "logging.hh"
+#include "finally.hh"
 
 #include "fetch-settings.hh"
 
@@ -314,6 +316,9 @@ struct GitInputScheme : InputScheme
         runProgram("git", true,
             { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) });
 
+        // Pause the logger to allow for user input (such as a gpg passphrase) in `git commit`
+        logger->pause();
+        Finally restoreLogger([]() { logger->resume(); });
         if (commitMsg)
             runProgram("git", true,
                 { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-m", *commitMsg });

From b7982372d234b1fd15bab01d09093471c1870bb4 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 19 Nov 2023 15:05:21 +0100
Subject: [PATCH 075/421] Compile hand-written release notes with changelog-d

---
 .gitignore                             |  1 +
 doc/manual/local.mk                    |  5 +++-
 doc/manual/rl-next/config              |  2 ++
 doc/manual/src/contributing/hacking.md | 38 ++++++++++++++++++++++++++
 flake.nix                              | 11 +++++++-
 5 files changed, 55 insertions(+), 2 deletions(-)
 create mode 100644 doc/manual/rl-next/config

diff --git a/.gitignore b/.gitignore
index 04d96ca2c..38e7e2b5a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,6 +27,7 @@ perl/Makefile.config
 /doc/manual/src/contributing/experimental-feature-descriptions.md
 /doc/manual/src/language/builtins.md
 /doc/manual/src/language/builtin-constants.md
+/doc/manual/src/release-notes/rl-next.md
 
 # /scripts/
 /scripts/nix-profile.sh
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index db3daf252..74a4103b3 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -144,6 +144,9 @@ $(d)/language.json: $(bindir)/nix
 	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp
 	@mv $@.tmp $@
 
+$(d)/src/release-notes/rl-next.md: $(d)/rl-next/*
+	$(trace-gen) changelog-d doc/manual/rl-next > $@
+
 # Generate the HTML manual.
 .PHONY: manual-html
 manual-html: $(docdir)/manual/index.html
@@ -177,7 +180,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
 # `@docroot@` is to be preserved for documenting the mechanism
 # FIXME: maybe contributing guides should live right next to the code
 # instead of in the manual
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md
 	$(trace-gen) \
 		tmp="$$(mktemp -d)"; \
 		cp -r doc/manual "$$tmp"; \
diff --git a/doc/manual/rl-next/config b/doc/manual/rl-next/config
new file mode 100644
index 000000000..b3c2e868f
--- /dev/null
+++ b/doc/manual/rl-next/config
@@ -0,0 +1,2 @@
+organization: NixOS
+repository: nix
diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index fe08ceb94..855900d7a 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -220,3 +220,41 @@ Configure your editor to use the `clangd` from the shell, either by running it i
 > For some editors (e.g. Visual Studio Code), you may need to install a [special extension](https://open-vsx.org/extension/llvm-vs-code-extensions/vscode-clangd) for the editor to interact with `clangd`.
 > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim).
 > Editor-specific setup is typically opinionated, so we will not cover it here in more detail.
+
+## Add a release note
+
+`doc/manual/rl-next` contains release notes entries for all unreleased changes.
+
+User-visible changes should come with a release note.
+
+### Add an entry
+
+Here's what a complete entry looks like. The file name is not incorporated in the document.
+
+```
+synopsis: Basically a title
+issues: #1234
+prs: #1238
+description: {
+
+Here's one or more paragraphs that describe the change.
+
+- It's markdown
+- Add references to the manual using @docroot@
+
+}
+```
+
+Significant changes should add the following header, which moves them to the top.
+
+```
+significance: significant
+```
+
+<!-- Keep an eye on https://codeberg.org/fgaz/changelog-d/issues/1 -->
+See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog).
+
+### Build process
+
+Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`.
+Non-releases build the release notes on the fly.
diff --git a/flake.nix b/flake.nix
index 9030a74f7..ca37948a7 100644
--- a/flake.nix
+++ b/flake.nix
@@ -173,6 +173,10 @@
           "--enable-internal-api-docs"
         ];
 
+        # TODO: after backport of https://github.com/NixOS/nixpkgs/pull/268487, remove `haskellPackages.` -
+        #       vastly improves output closure, and adds shell completions
+        changelog-d = pkgs.buildPackages.haskellPackages.changelog-d;
+
         nativeBuildDeps =
           [
             buildPackages.bison
@@ -190,7 +194,10 @@
             buildPackages.jq # Also for custom mdBook preprocessor.
             buildPackages.openssh # only needed for tests (ssh-keygen)
           ]
-          ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)];
+          ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]
+          # Official releases don't have rl-next, so we don't need to compile a changelog
+          ++ lib.optional (!officialRelease) changelog-d
+          ;
 
         buildDeps =
           [ curl
@@ -727,6 +734,8 @@
               ++ lib.optional
                 (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform)
                 pkgs.buildPackages.clang-tools
+              # We want changelog-d in the shell even if it's an official release
+              ++ lib.optional officialRelease changelog-d
               ;
 
             buildInputs = buildDeps ++ propagatedDeps

From b26038c517ed10feae751ad6733244c00b715d34 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 19 Nov 2023 16:08:34 +0100
Subject: [PATCH 076/421] doc: Rename 2X.XX to "Upcoming release", and only
 generate if applicable

---
 .gitignore                   |  1 +
 doc/manual/local.mk          | 13 +++++++++++--
 doc/manual/src/SUMMARY.md.in |  2 +-
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/.gitignore b/.gitignore
index 38e7e2b5a..5c359d7a0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,6 +21,7 @@ perl/Makefile.config
 /doc/manual/language.json
 /doc/manual/xp-features.json
 /doc/manual/src/SUMMARY.md
+/doc/manual/src/SUMMARY-rl-next.md
 /doc/manual/src/command-ref/new-cli
 /doc/manual/src/command-ref/conf-file.md
 /doc/manual/src/command-ref/experimental-features-shortlist.md
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index 74a4103b3..265a4649d 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -92,7 +92,7 @@ $(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md
 	$(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@
 	@rm $^.tmp
 
-$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
+$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
 	@cp $< $@
 	@$(call process-includes,$@,$@)
 
@@ -144,9 +144,18 @@ $(d)/language.json: $(bindir)/nix
 	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp
 	@mv $@.tmp $@
 
-$(d)/src/release-notes/rl-next.md: $(d)/rl-next/*
+# Generate "Upcoming release" notes (or clear it and remove from menu)
+$(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/*
 	$(trace-gen) changelog-d doc/manual/rl-next > $@
 
+$(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md
+	$(trace-gen) true
+	@if [ -s $< ]; then \
+		echo '  - [Upcoming release](release-notes/rl-next.md)' > $@; \
+	else \
+	  true > $@; \
+	fi
+
 # Generate the HTML manual.
 .PHONY: manual-html
 manual-html: $(docdir)/manual/index.html
diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in
index 8dc464abd..8e7b4eeab 100644
--- a/doc/manual/src/SUMMARY.md.in
+++ b/doc/manual/src/SUMMARY.md.in
@@ -114,7 +114,7 @@
   - [CLI guideline](contributing/cli-guideline.md)
   - [C++ style guide](contributing/cxx.md)
 - [Release Notes](release-notes/release-notes.md)
-  - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
+{{#include ./SUMMARY-rl-next.md}}
   - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md)
   - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md)
   - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md)

From 9aa63f70d7d861ba74764188410c0add730cd48d Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 24 Nov 2023 15:32:02 +0100
Subject: [PATCH 077/421] fricklerhandwerk: subscribe to documentation changes
 (#9422)

* fricklerhandwerk: subscribe to documentation changes
---
 .github/CODEOWNERS | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index ab5908649..39d595199 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -14,5 +14,12 @@
 /doc @fricklerhandwerk
 *.md @fricklerhandwerk
 
+# Documentation of built-in functions
+src/libexpr/primops.cc @fricklerhandwerk @roberth
+# Documentation on experimental features
+src/libutil/experimental-features.cc @fricklerhandwerk
+# Documentation on configuration settings
+src/libstore/globals.hh @fricklerhandwerk
+
 # Libstore layer
 /src/libstore @thufschmitt

From 2a538c571b13877fa426f2cff2749cf17d140216 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 19 Nov 2023 16:49:11 +0100
Subject: [PATCH 078/421] Add scripts/release-notes

---
 maintainers/release-process.md |  45 ++++------
 scripts/release-notes          | 148 +++++++++++++++++++++++++++++++++
 2 files changed, 166 insertions(+), 27 deletions(-)
 create mode 100755 scripts/release-notes

diff --git a/maintainers/release-process.md b/maintainers/release-process.md
index d85266b81..e542d8be5 100644
--- a/maintainers/release-process.md
+++ b/maintainers/release-process.md
@@ -24,34 +24,23 @@ release:
 * In a checkout of the Nix repo, make sure you're on `master` and run
   `git pull`.
 
-* Move the contents of `doc/manual/src/release-notes/rl-next.md`
-  (except the first line) to
-  `doc/manual/src/release-notes/rl-$VERSION.md` (where `$VERSION` is
-  the contents of `.version` *without* the patch level, e.g. `2.12`
-  rather than `2.12.0`).
-
-* Add a header to `doc/manual/src/release-notes/rl-$VERSION.md` like
-
-  ```
-  # Release 2.12 (2022-12-06)
-  ```
-
-* Proof-read / edit / rearrange the release notes. Breaking changes
-  and highlights should go to the top.
-
-* Add a link to the release notes to `doc/manual/src/SUMMARY.md.in`
-  (*not* `SUMMARY.md`), e.g.
-
-  ```
-  - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md)
-  ```
-
-* Run
+* Compile the release notes by running
 
   ```console
   $ git checkout -b release-notes
-  $ git add doc/manual/src/release-notes/rl-$VERSION.md
-  $ git commit -a -m 'Release notes'
+  $ VERSION=X.YY ./scripts/release-notes
+  ```
+
+  where `X.YY` is *without* the patch level, e.g. `2.12` rather than ~~`2.12.0`~~.
+
+  A commit is created.
+
+* Proof-read / edit / rearrange the release notes if needed. Breaking changes
+  and highlights should go to the top.
+
+* Push.
+
+  ```console
   $ git push --set-upstream $REMOTE release-notes
   ```
 
@@ -67,15 +56,17 @@ release:
   $ git checkout -b $VERSION-maintenance
   ```
 
-* Mark the release as stable:
+* Mark the release as official:
 
   ```console
-  $ git cherry-pick f673551e71942a52b6d7ae66af8b67140904a76a
+  $ sed -e 's/officialRelease = false;/officialRelease = true;/' -i flake.nix
   ```
 
   This removes the link to `rl-next.md` from the manual and sets
   `officialRelease = true` in `flake.nix`.
 
+* Commit
+
 * Push the release branch:
 
   ```console
diff --git a/scripts/release-notes b/scripts/release-notes
new file mode 100755
index 000000000..e5ee39d11
--- /dev/null
+++ b/scripts/release-notes
@@ -0,0 +1,148 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash ../shell.nix -I nixpkgs=channel:nixos-unstable-small
+#                                   ^^^^^^^
+#                Only used for bash. shell.nix goes to the flake.
+
+# --- CONFIGURATION ---
+
+# This does double duty for
+# - including rl-next
+# - marking where to insert new links (right after)
+SUMMARY_MARKER_LINE='{{#include ./SUMMARY-rl-next.md}}'
+
+# --- LIB ---
+
+log() {
+  echo 1>&2 "release-notes:" "$@"
+}
+logcmd() {
+  local cmd="$1"
+  shift
+  logcmd2 "$cmd" "${*@Q}" "$cmd" "$@"
+}
+logcmd2() {
+  local fakecmd="$1"
+  local fakeargs="$2"
+  shift
+  shift
+  printf 1>&2 "release-notes: \033[34;1m$fakecmd\033[0m "
+  echo "$fakeargs" 1>&2
+  "$@"
+}
+die() {
+  # ANSI red
+  printf 1>&2 "release-notes: \033[31;1merror:\033[0m"
+  echo 1>&2 "" "$@"
+  exit 1
+}
+confirm() {
+  local answer
+  echo 1>&2 "$@" "[y/n]"
+  read -r answer
+  case "$answer" in
+    y|Y|yes|Yes|YES)
+      return 0
+      ;;
+    n|N|no|No|NO)
+      return 1
+      ;;
+    *)
+      echo 1>&2 "please answer y or n"
+      confirm "$@"
+      ;;
+  esac
+}
+report_done() {
+  logcmd2 "git" "show" git -c pager.show=false show
+  printf 1>&2 "release-notes: \033[32;1mdone\033[0m\n"
+}
+
+# --- PARSE ARGS ---
+
+if [[ $# -gt 0 ]]; then
+  die "Release notes takes no arguments, but make sure to set VERSION."
+fi
+
+# --- CHECKS ---
+
+if [[ ! -e flake.nix ]] || [[ ! -e .git ]]; then
+  die "must run in repo root"
+  exit 1
+fi
+
+# repo must be clean
+if ! git diff --quiet; then
+  die "repo is dirty, please commit or stash changes"
+fi
+
+if ! git diff --quiet --cached; then
+  die "repo has staged changes, please commit or stash them"
+fi
+
+if ! grep "$SUMMARY_MARKER_LINE" doc/manual/src/SUMMARY.md.in >/dev/null; then
+  # would have been nice to catch this early, but won't be worth the extra infra
+  die "SUMMARY.md.in is missing the marker line '$SUMMARY_MARKER_LINE', which would be used for inserting a new release notes page. Please fix the script."
+fi
+
+if [[ ! -n "${VERSION:-}" ]]; then
+  die "please set the VERSION environment variable before invoking this script"
+  exit 1
+fi
+
+case "$VERSION" in
+  # FIXME: accepts "." without any real digits
+  [[:digit:]]*.[[:digit:]]*)
+    ;;
+  *)
+    die "VERSION must be MAJOR.MINOR, where each is a number, e.g. 2.20 (VERSION was set to $VERSION)"
+    ;;
+esac
+
+# --- DEFAULTS ---
+
+if [[ ! -n "${DATE:-}" ]]; then
+  DATE="$(date +%Y-%m-%d)"
+  log "DATE not set, using $DATE"
+fi
+
+case "$DATE" in
+  [[:digit:]]*-[[:digit:]]*-[[:digit:]]*)
+    ;;
+  *)
+    die "DATE must be YYYY-MM-DD, e.g. 2021-12-31 (DATE was set to $DATE)"
+    ;;
+esac
+
+# --- DO THE WORK ---
+
+basename=rl-$VERSION.md
+file=doc/manual/src/release-notes/$basename
+title="Release $VERSION ($DATE)"
+
+(
+  # TODO add minor number, and append?
+  echo "# $title"
+  echo
+  changelog-d doc/manual/rl-next | sed -e 's/ *$//'
+) > $file
+
+log "Wrote $file"
+
+NEW_SUMMARY_LINE="  - [$title](release-notes/$basename)"
+
+# find the marker line, insert new link after it
+escaped_marker="$(echo "$SUMMARY_MARKER_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')"
+escaped_line="$(echo "$NEW_SUMMARY_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')"
+logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/src/SUMMARY.md.in
+
+for f in doc/manual/rl-next/*.md; do
+  if [[ config != "$(basename $f)" ]]; then
+    logcmd git rm $f
+  fi
+done
+
+logcmd git add $file doc/manual/src/SUMMARY.md.in
+logcmd git status
+logcmd git commit -m "release notes: $VERSION"
+
+report_done

From b1ea30f21d24df9afc4eb1635eee9a080e4f81f3 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 19 Nov 2023 18:13:59 +0100
Subject: [PATCH 079/421] scripts/release-notes: Support patch releases

This also fixes the broken case statement, which has globs, not regexes.
---
 maintainers/release-process.md | 24 +++++++++++++
 scripts/release-notes          | 65 ++++++++++++++++++++++++----------
 2 files changed, 71 insertions(+), 18 deletions(-)

diff --git a/maintainers/release-process.md b/maintainers/release-process.md
index e542d8be5..b1259a9bd 100644
--- a/maintainers/release-process.md
+++ b/maintainers/release-process.md
@@ -150,6 +150,30 @@ release:
 
 ## Creating a point release
 
+* Checkout.
+
+  ```console
+  $ git checkout XX.YY-maintenance
+  ```
+
+* Determine the next patch version.
+
+  ```console
+  $ export VERSION=XX.YY.ZZ
+  ```
+
+* Update release notes.
+
+  ```console
+  $ ./scripts/release-notes
+  ```
+
+* Push.
+
+  ```console
+  $ git push
+  ```
+
 * Wait for the desired evaluation of the maintenance jobset to finish
   building.
 
diff --git a/scripts/release-notes b/scripts/release-notes
index e5ee39d11..43f20a547 100755
--- a/scripts/release-notes
+++ b/scripts/release-notes
@@ -89,14 +89,40 @@ if [[ ! -n "${VERSION:-}" ]]; then
   exit 1
 fi
 
-case "$VERSION" in
-  # FIXME: accepts "." without any real digits
-  [[:digit:]]*.[[:digit:]]*)
-    ;;
-  *)
-    die "VERSION must be MAJOR.MINOR, where each is a number, e.g. 2.20 (VERSION was set to $VERSION)"
-    ;;
-esac
+# mutate/initialize:
+# VERSION:      MAJOR.MINOR
+# FULL_VERSION: MAJOR.MINOR.PATCH
+# IS_PATCH:     true if this is a patch release; append instead of create
+if grep -E '^[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then
+  log 'is minor'
+  IS_PATCH=false
+  FULL_VERSION="$VERSION.0"
+elif grep -E '^[0-9]+\.[0-9]+\.0$' <<< "$VERSION" >/dev/null; then
+  log 'is minor (.0)'
+  IS_PATCH=false
+  FULL_VERSION="$VERSION"
+  VERSION="$(echo "$VERSION" | sed -e 's/\.0$//')"
+elif grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then
+  log 'is patch'
+  IS_PATCH=true
+  FULL_VERSION="$VERSION"
+  VERSION="$(echo "$VERSION" | sed -e 's/\.[0-9]*$//')"
+else
+  die "VERSION must be MAJOR.MINOR[.PATCH], where each is a number, e.g. 2.20 or 2.20.1 (VERSION was set to $VERSION)"
+fi
+
+log "VERSION=$VERSION"
+log "FULL_VERSION=$FULL_VERSION"
+log "IS_PATCH=$IS_PATCH"
+
+basename=rl-$VERSION.md
+file=doc/manual/src/release-notes/$basename
+
+if ! $IS_PATCH; then
+  if [[ -e $file ]]; then
+    die "release notes file $file already exists. If you'd like to make a minor release, pass a patch version, e.g. 2.20.1"
+  fi
+fi
 
 # --- DEFAULTS ---
 
@@ -106,7 +132,7 @@ if [[ ! -n "${DATE:-}" ]]; then
 fi
 
 case "$DATE" in
-  [[:digit:]]*-[[:digit:]]*-[[:digit:]]*)
+  [0-9]*-[0-9]*-[0-9]*)
     ;;
   *)
     die "DATE must be YYYY-MM-DD, e.g. 2021-12-31 (DATE was set to $DATE)"
@@ -115,25 +141,28 @@ esac
 
 # --- DO THE WORK ---
 
-basename=rl-$VERSION.md
-file=doc/manual/src/release-notes/$basename
+# menu
 title="Release $VERSION ($DATE)"
+# section on page
+section_title="Release $FULL_VERSION ($DATE)"
 
 (
   # TODO add minor number, and append?
-  echo "# $title"
+  echo "# $section_title"
   echo
   changelog-d doc/manual/rl-next | sed -e 's/ *$//'
-) > $file
+) | tee -a $file
 
 log "Wrote $file"
 
-NEW_SUMMARY_LINE="  - [$title](release-notes/$basename)"
+if ! $IS_PATCH; then
+  NEW_SUMMARY_LINE="  - [$title](release-notes/$basename)"
 
-# find the marker line, insert new link after it
-escaped_marker="$(echo "$SUMMARY_MARKER_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')"
-escaped_line="$(echo "$NEW_SUMMARY_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')"
-logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/src/SUMMARY.md.in
+  # find the marker line, insert new link after it
+  escaped_marker="$(echo "$SUMMARY_MARKER_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')"
+  escaped_line="$(echo "$NEW_SUMMARY_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')"
+  logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/src/SUMMARY.md.in
+fi
 
 for f in doc/manual/rl-next/*.md; do
   if [[ config != "$(basename $f)" ]]; then

From 7c4ee5c8135fae65602791f0b89d0dbae7e94f3e Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 19 Nov 2023 18:36:50 +0100
Subject: [PATCH 080/421] scripts/release-notes: Avoid mutating variables

---
 scripts/release-notes | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/scripts/release-notes b/scripts/release-notes
index 43f20a547..2e1be64a4 100755
--- a/scripts/release-notes
+++ b/scripts/release-notes
@@ -89,33 +89,35 @@ if [[ ! -n "${VERSION:-}" ]]; then
   exit 1
 fi
 
-# mutate/initialize:
-# VERSION:      MAJOR.MINOR
-# FULL_VERSION: MAJOR.MINOR.PATCH
-# IS_PATCH:     true if this is a patch release; append instead of create
+# version_major_minor:      MAJOR.MINOR
+# version_full:             MAJOR.MINOR.PATCH
+# IS_PATCH:                 true if this is a patch release; append instead of create
 if grep -E '^[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then
   log 'is minor'
   IS_PATCH=false
-  FULL_VERSION="$VERSION.0"
+  version_full="$VERSION.0"
+  version_major_minor="$VERSION"
 elif grep -E '^[0-9]+\.[0-9]+\.0$' <<< "$VERSION" >/dev/null; then
   log 'is minor (.0)'
   IS_PATCH=false
-  FULL_VERSION="$VERSION"
-  VERSION="$(echo "$VERSION" | sed -e 's/\.0$//')"
+  version_full="$VERSION"
+  version_major_minor="$(echo "$VERSION" | sed -e 's/\.0$//')"
 elif grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then
   log 'is patch'
   IS_PATCH=true
-  FULL_VERSION="$VERSION"
-  VERSION="$(echo "$VERSION" | sed -e 's/\.[0-9]*$//')"
+  version_full="$VERSION"
+  version_major_minor="$(echo "$VERSION" | sed -e 's/\.[0-9]*$//')"
 else
   die "VERSION must be MAJOR.MINOR[.PATCH], where each is a number, e.g. 2.20 or 2.20.1 (VERSION was set to $VERSION)"
 fi
 
-log "VERSION=$VERSION"
-log "FULL_VERSION=$FULL_VERSION"
+unset VERSION
+
+log "version_major_minor=$version_major_minor"
+log "version_full=$version_full"
 log "IS_PATCH=$IS_PATCH"
 
-basename=rl-$VERSION.md
+basename=rl-${version_major_minor}.md
 file=doc/manual/src/release-notes/$basename
 
 if ! $IS_PATCH; then
@@ -142,9 +144,9 @@ esac
 # --- DO THE WORK ---
 
 # menu
-title="Release $VERSION ($DATE)"
+title="Release $version_major_minor ($DATE)"
 # section on page
-section_title="Release $FULL_VERSION ($DATE)"
+section_title="Release $version_full ($DATE)"
 
 (
   # TODO add minor number, and append?
@@ -172,6 +174,6 @@ done
 
 logcmd git add $file doc/manual/src/SUMMARY.md.in
 logcmd git status
-logcmd git commit -m "release notes: $VERSION"
+logcmd git commit -m "release notes: $version_full"
 
 report_done

From 6971c4adc06d574cbe1e9ab6da19814e11e2ba6c Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sun, 19 Nov 2023 19:08:42 +0100
Subject: [PATCH 081/421] maintainers/release-notes <- scripts/release-notes

---
 {scripts => maintainers}/release-notes | 0
 maintainers/release-process.md         | 4 ++--
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename {scripts => maintainers}/release-notes (100%)

diff --git a/scripts/release-notes b/maintainers/release-notes
similarity index 100%
rename from scripts/release-notes
rename to maintainers/release-notes
diff --git a/maintainers/release-process.md b/maintainers/release-process.md
index b1259a9bd..db8b064a5 100644
--- a/maintainers/release-process.md
+++ b/maintainers/release-process.md
@@ -28,7 +28,7 @@ release:
 
   ```console
   $ git checkout -b release-notes
-  $ VERSION=X.YY ./scripts/release-notes
+  $ VERSION=X.YY ./maintainers/release-notes
   ```
 
   where `X.YY` is *without* the patch level, e.g. `2.12` rather than ~~`2.12.0`~~.
@@ -165,7 +165,7 @@ release:
 * Update release notes.
 
   ```console
-  $ ./scripts/release-notes
+  $ ./maintainers/release-notes
   ```
 
 * Push.

From 857f9168f7b48aa491052f24fb571c21398f9826 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 24 Nov 2023 15:27:54 +0100
Subject: [PATCH 082/421] Migrate rl-next.md to doc/manual/rl-next directory

---
 doc/manual/rl-next/mounted-ssh-store.md     | 9 +++++++++
 doc/manual/rl-next/nix-env-json-drv-path.md | 9 +++++++++
 doc/manual/src/release-notes/rl-next.md     | 6 ------
 3 files changed, 18 insertions(+), 6 deletions(-)
 create mode 100644 doc/manual/rl-next/mounted-ssh-store.md
 create mode 100644 doc/manual/rl-next/nix-env-json-drv-path.md
 delete mode 100644 doc/manual/src/release-notes/rl-next.md

diff --git a/doc/manual/rl-next/mounted-ssh-store.md b/doc/manual/rl-next/mounted-ssh-store.md
new file mode 100644
index 000000000..39fac5283
--- /dev/null
+++ b/doc/manual/rl-next/mounted-ssh-store.md
@@ -0,0 +1,9 @@
+synopsis: Mounted SSH Store
+issues: #7890
+prs: #7912
+description: {
+
+Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
+This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem.
+
+}
diff --git a/doc/manual/rl-next/nix-env-json-drv-path.md b/doc/manual/rl-next/nix-env-json-drv-path.md
new file mode 100644
index 000000000..fbe2b67d8
--- /dev/null
+++ b/doc/manual/rl-next/nix-env-json-drv-path.md
@@ -0,0 +1,9 @@
+synopsis: Fix `nix-env --query --drv-path --json`
+prs: #9257
+description: {
+
+Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.
+
+}
+
+
diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md
deleted file mode 100644
index 0e3d8b462..000000000
--- a/doc/manual/src/release-notes/rl-next.md
+++ /dev/null
@@ -1,6 +0,0 @@
-# Release X.Y (202?-??-??)
-
-- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.
-
-- Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
-  This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem.

From 54b684765519dee1e74dca71605f8e7f6c8b0a25 Mon Sep 17 00:00:00 2001
From: ivan770 <ivan@ivan770.me>
Date: Fri, 24 Nov 2023 11:17:35 -0500
Subject: [PATCH 083/421] doc: fix machine-specific capabilities leaking

---
 src/libstore/globals.hh   | 14 +++++++++++---
 src/libstore/store-api.hh |  5 ++++-
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 27caf42c4..838d2aba2 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -183,7 +183,9 @@ public:
           command line switch and defaults to `1`. The value `0` means that
           the builder should use all available CPU cores in the system.
         )",
-        {"build-cores"}, false};
+        {"build-cores"},
+        // Don't document the machine-specific default value
+        false};
 
     /**
      * Read-only mode.  Don't copy stuff to the store, don't change
@@ -699,7 +701,10 @@ public:
 
           Build systems will usually detect the target platform to be the current physical system and therefore produce machine code incompatible with what may be intended in the derivation.
           You should design your derivation's `builder` accordingly and cross-check the results when using this option against natively-built versions of your derivation.
-        )", {}, false};
+        )",
+        {},
+        // Don't document the machine-specific default value
+        false};
 
     Setting<StringSet> systemFeatures{
         this,
@@ -744,7 +749,10 @@ public:
             [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix.
 
             Included by default on Linux if the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting is enabled.
-        )", {}, false};
+        )",
+        {},
+        // Don't document the machine-specific default value
+        false};
 
     Setting<Strings> substituters{
         this,
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 32ad2aa44..8b6bf9aed 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -167,7 +167,10 @@ struct StoreConfig : public Config
           Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations.
 
           Example: `"kvm"`
-        )" };
+        )",
+        {},
+        // Don't document the machine-specific default value
+        false};
 };
 
 class Store : public std::enable_shared_from_this<Store>, public virtual StoreConfig

From d2f5e263e3c095dfe9d874387665b88c4bfff6f1 Mon Sep 17 00:00:00 2001
From: Yorick van Pelt <yorick@yorickvanpelt.nl>
Date: Mon, 6 Feb 2023 16:36:57 +0100
Subject: [PATCH 084/421] Switch from std::regex to boost::regex

---
 flake.nix              | 24 +++++++++++++++++-------
 src/libexpr/local.mk   |  2 +-
 src/libexpr/primops.cc | 35 +++++++++++++++++++++++------------
 3 files changed, 41 insertions(+), 20 deletions(-)

diff --git a/flake.nix b/flake.nix
index 9030a74f7..1f7e7b2af 100644
--- a/flake.nix
+++ b/flake.nix
@@ -154,7 +154,7 @@
 
         configureFlags =
           lib.optionals stdenv.isLinux [
-            "--with-boost=${boost}/lib"
+            "--with-boost=${boost-nix}/lib"
             "--with-sandbox-shell=${sh}/bin/busybox"
           ]
           ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [
@@ -202,7 +202,7 @@
               version = libgit2.lastModifiedDate;
               cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"];
             }))
-            boost
+            boost-nix
             lowdown-nix
             libsodium
           ]
@@ -423,14 +423,14 @@
 
             propagatedBuildInputs = propagatedDeps;
 
-            disallowedReferences = [ boost ];
+            disallowedReferences = [ boost-nix ];
 
             preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic)
               ''
                 # Copy libboost_context so we don't get all of Boost in our closure.
                 # https://github.com/NixOS/nixpkgs/issues/45462
                 mkdir -p $out/lib
-                cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
+                cp -pd ${boost-nix}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib
                 rm -f $out/lib/*.a
                 ${lib.optionalString currentStdenv.hostPlatform.isLinux ''
                   chmod u+w $out/lib/*.so.*
@@ -440,9 +440,9 @@
                   for LIB in $out/lib/*.dylib; do
                     chmod u+w $LIB
                     install_name_tool -id $LIB $LIB
-                    install_name_tool -delete_rpath ${boost}/lib/ $LIB || true
+                    install_name_tool -delete_rpath ${boost-nix}/lib/ $LIB || true
                   done
-                  install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
+                  install_name_tool -change ${boost-nix}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
                 ''}
               '';
 
@@ -470,9 +470,13 @@
               ''}
               ${lib.optionalString currentStdenv.isDarwin ''
               install_name_tool \
-                -change ${boost}/lib/libboost_context.dylib \
+                -change ${boost-nix}/lib/libboost_context.dylib \
                 $out/lib/libboost_context.dylib \
                 $out/lib/libnixutil.dylib
+              install_name_tool \
+                -change ${boost-nix}/lib/libboost_regex.dylib \
+                $out/lib/libboost_regex.dylib \
+                $out/lib/libnixexpr.dylib
               ''}
             '';
 
@@ -495,6 +499,12 @@
             meta.mainProgram = "nix";
           });
 
+          boost-nix = final.boost.override {
+            # enableIcu arg is not yet supported
+            # but will be with next nixpkgs update
+            enableIcu = false;
+          };
+
           lowdown-nix = with final; currentStdenv.mkDerivation rec {
             name = "lowdown-0.9.0";
 
diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index ed7bf9490..946059339 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -16,7 +16,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib
 
 libexpr_LIBS = libutil libstore libfetchers
 
-libexpr_LDFLAGS += -lboost_context -pthread
+libexpr_LDFLAGS += -lboost_context -lboost_regex -pthread
 ifdef HOST_LINUX
  libexpr_LDFLAGS += -ldl
 endif
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index ba735b435..0c34cb6e8 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -16,6 +16,7 @@
 #include "primops.hh"
 
 #include <boost/container/small_vector.hpp>
+#include <boost/regex.hpp>
 #include <nlohmann/json.hpp>
 
 #include <sys/types.h>
@@ -24,7 +25,6 @@
 
 #include <algorithm>
 #include <cstring>
-#include <regex>
 #include <dlfcn.h>
 
 #include <cmath>
@@ -3885,19 +3885,30 @@ static RegisterPrimOp primop_convertHash({
     .fun = prim_convertHash,
 });
 
+// regex aliases, switch between boost and std
+using regex = boost::regex;
+using regex_error = boost::regex_error;
+using cmatch = boost::cmatch;
+using cregex_iterator = boost::cregex_iterator;
+namespace regex_constants = boost::regex_constants;
+// overloaded function alias
+constexpr auto regex_match = [] (auto &&...args) {
+    return boost::regex_match(std::forward<decltype(args)>(args)...);
+ };
+
 struct RegexCache
 {
     // TODO use C++20 transparent comparison when available
-    std::unordered_map<std::string_view, std::regex> cache;
+    std::unordered_map<std::string_view, regex> cache;
     std::list<std::string> keys;
 
-    std::regex get(std::string_view re)
+    regex get(std::string_view re)
     {
         auto it = cache.find(re);
         if (it != cache.end())
             return it->second;
         keys.emplace_back(re);
-        return cache.emplace(keys.back(), std::regex(keys.back(), std::regex::extended)).first->second;
+        return cache.emplace(keys.back(), regex(keys.back(), regex::extended)).first->second;
     }
 };
 
@@ -3917,8 +3928,8 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v)
         NixStringContext context;
         const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.match");
 
-        std::cmatch match;
-        if (!std::regex_match(str.begin(), str.end(), match, regex)) {
+        cmatch match;
+        if (!regex_match(str.begin(), str.end(), match, regex)) {
             v.mkNull();
             return;
         }
@@ -3933,8 +3944,8 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v)
                 (v.listElems()[i] = state.allocValue())->mkString(match[i + 1].str());
         }
 
-    } catch (std::regex_error & e) {
-        if (e.code() == std::regex_constants::error_space) {
+    } catch (regex_error & e) {
+        if (e.code() == regex_constants::error_space) {
             // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
             state.debugThrowLastTrace(EvalError({
                 .msg = hintfmt("memory limit exceeded by regular expression '%s'", re),
@@ -3997,8 +4008,8 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v)
         NixStringContext context;
         const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.split");
 
-        auto begin = std::cregex_iterator(str.begin(), str.end(), regex);
-        auto end = std::cregex_iterator();
+        auto begin = cregex_iterator(str.begin(), str.end(), regex);
+        auto end = cregex_iterator();
 
         // Any matches results are surrounded by non-matching results.
         const size_t len = std::distance(begin, end);
@@ -4037,8 +4048,8 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 
         assert(idx == 2 * len + 1);
 
-    } catch (std::regex_error & e) {
-        if (e.code() == std::regex_constants::error_space) {
+    } catch (regex_error & e) {
+        if (e.code() == regex_constants::error_space) {
             // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
             state.debugThrowLastTrace(EvalError({
                 .msg = hintfmt("memory limit exceeded by regular expression '%s'", re),

From 213594721ace2b2e2dcf3d57615178a2c1690aa4 Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Sat, 25 Nov 2023 17:21:49 +0100
Subject: [PATCH 085/421] gitignore: Also ignore .DS_Store

This is a file that Finder on Mac OS loves to add into various folders.
---
 .gitignore | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitignore b/.gitignore
index 04d96ca2c..f7dcea422 100644
--- a/.gitignore
+++ b/.gitignore
@@ -144,3 +144,6 @@ result
 
 # clangd and possibly more
 .cache/
+
+# Mac OS
+.DS_Store

From f25c06d7a3289343887d09761c82356a3b6b441b Mon Sep 17 00:00:00 2001
From: Jan Tojnar <jtojnar@gmail.com>
Date: Sun, 26 Nov 2023 15:46:09 +0100
Subject: [PATCH 086/421] docs: Fix broken link

Link target definitions need to be in a separate paragraph to be collected.

Fixup for https://github.com/NixOS/nix/commit/217d863f7a251a4d8a08ff3294944b45146c61c9
---
 doc/manual/src/language/values.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md
index 0bb656746..aea68a441 100644
--- a/doc/manual/src/language/values.md
+++ b/doc/manual/src/language/values.md
@@ -116,6 +116,7 @@
   [store path]: ../glossary.md#gloss-store-path
 
   Paths can include [string interpolation] and can themselves be [interpolated in other expressions].
+
   [interpolated in other expressions]: ./string-interpolation.md#interpolated-expressions
 
   At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path.

From d63f72197cec6ff95d9ffc83aa8076acd86a3fd1 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 24 Nov 2023 15:48:41 +0100
Subject: [PATCH 087/421] Don't run changelog-d in the build

This way we lose the preview of release notes on master, as well
as on https://nixos.org/manual/nix/unstable/release-notes/rl-next
but we can come back to this.
---
 doc/manual/local.mk                    | 8 +++++++-
 doc/manual/src/contributing/hacking.md | 2 +-
 flake.nix                              | 9 ++++++---
 3 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index 265a4649d..f22dfa69e 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -146,7 +146,13 @@ $(d)/language.json: $(bindir)/nix
 
 # Generate "Upcoming release" notes (or clear it and remove from menu)
 $(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/*
-	$(trace-gen) changelog-d doc/manual/rl-next > $@
+	@if type -p changelog-d > /dev/null; then \
+		echo "  GEN   " $@; \
+		changelog-d doc/manual/rl-next > $@; \
+	else \
+		echo "  NULL  " $@; \
+		true > $@; \
+	fi
 
 $(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md
 	$(trace-gen) true
diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index 855900d7a..3291d5a20 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -257,4 +257,4 @@ See also the [format documentation](https://github.com/haskell/cabal/blob/master
 ### Build process
 
 Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`.
-Non-releases build the release notes on the fly.
+Set `buildUnreleasedNotes = true;` in `flake.nix` to build the release notes on the fly.
diff --git a/flake.nix b/flake.nix
index ca37948a7..db913f062 100644
--- a/flake.nix
+++ b/flake.nix
@@ -14,6 +14,9 @@
 
       officialRelease = false;
 
+      # Set to true to build the release notes for the next release.
+      buildUnreleasedNotes = false;
+
       version = lib.fileContents ./.version + versionSuffix;
       versionSuffix =
         if officialRelease
@@ -196,7 +199,7 @@
           ]
           ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]
           # Official releases don't have rl-next, so we don't need to compile a changelog
-          ++ lib.optional (!officialRelease) changelog-d
+          ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d
           ;
 
         buildDeps =
@@ -734,8 +737,8 @@
               ++ lib.optional
                 (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform)
                 pkgs.buildPackages.clang-tools
-              # We want changelog-d in the shell even if it's an official release
-              ++ lib.optional officialRelease changelog-d
+              # We want changelog-d in the shell even if the current build doesn't need it
+              ++ lib.optional (officialRelease || ! buildUnreleasedNotes) changelog-d
               ;
 
             buildInputs = buildDeps ++ propagatedDeps

From c5d49ec7ab7b9fb33f0336a909ac837e208be807 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 24 Nov 2023 16:18:27 +0100
Subject: [PATCH 088/421] flake.lock: Update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/decdf666c833a325cb4417041a90681499e06a41' (2023-11-18)
  → 'github:NixOS/nixpkgs/9ba29e2346bc542e9909d1021e8fd7d4b3f64db0' (2023-11-23)
---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 1b3c0b6d5..f120d3b5f 100644
--- a/flake.lock
+++ b/flake.lock
@@ -50,11 +50,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1700342017,
-        "narHash": "sha256-HaibwlWH5LuqsaibW3sIVjZQtEM/jWtOHX4Nk93abGE=",
+        "lastModified": 1700748986,
+        "narHash": "sha256-/nqLrNU297h3PCw4QyDpZKZEUHmialJdZW2ceYFobds=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "decdf666c833a325cb4417041a90681499e06a41",
+        "rev": "9ba29e2346bc542e9909d1021e8fd7d4b3f64db0",
         "type": "github"
       },
       "original": {

From e7e21aa0c839460a62456fa44f31339c187077ff Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 24 Nov 2023 16:27:10 +0100
Subject: [PATCH 089/421] flake.nix: Use top level changelog-d

It is about 2 MB now, as only it and libffi (tiny) are new in the
build or shell closures.
---
 flake.nix | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/flake.nix b/flake.nix
index db913f062..a239226c0 100644
--- a/flake.nix
+++ b/flake.nix
@@ -176,9 +176,7 @@
           "--enable-internal-api-docs"
         ];
 
-        # TODO: after backport of https://github.com/NixOS/nixpkgs/pull/268487, remove `haskellPackages.` -
-        #       vastly improves output closure, and adds shell completions
-        changelog-d = pkgs.buildPackages.haskellPackages.changelog-d;
+        changelog-d = pkgs.buildPackages.changelog-d;
 
         nativeBuildDeps =
           [

From 06a745120bc8fe7625954e970c61028f8a42c31e Mon Sep 17 00:00:00 2001
From: Bob van der Linden <bobvanderlinden@gmail.com>
Date: Sun, 26 Nov 2023 21:27:46 +0100
Subject: [PATCH 090/421] nix: develop: remove test for interactive shell
 executable

---
 tests/functional/flakes/develop.sh | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/tests/functional/flakes/develop.sh b/tests/functional/flakes/develop.sh
index db23ca0c0..e1e53d364 100644
--- a/tests/functional/flakes/develop.sh
+++ b/tests/functional/flakes/develop.sh
@@ -50,14 +50,6 @@ EOF
 nix build --no-write-lock-file './nixpkgs#bashInteractive' --out-link ./bash-interactive
 BASH_INTERACTIVE_EXECUTABLE="$PWD/bash-interactive/bin/bash"
 
-# Test whether `nix develop` uses nixpkgs#bashInteractive shell.
-[[ "$(
-    nix develop --no-write-lock-file .#hello <<EOF
-export PATH="$PATH"
-$lsof -a -d txt -p \$$ 2>&1 | grep -o '/.*/bash'
-EOF
-)" -ef "$BASH_INTERACTIVE_EXECUTABLE" ]]
-
 # Test whether `nix develop` sets `SHELL` to nixpkgs#bashInteractive shell.
 [[ "$(
     SHELL=custom nix develop --no-write-lock-file .#hello <<EOF

From 2b7016cc56d12e67de9f1f25b18311866a26a5fe Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Mon, 27 Nov 2023 08:33:03 +0100
Subject: [PATCH 091/421] add path based redirects

up to now, those were managed outside of this repo, which as
unsurprisingly a real hassle to deal with if one wanted to prevent URLs
from breaking when moving pages around. this change removes a large part
of the friction involved in moving content in the Nix manual.

possible next steps for further automation:
- check for content that moved and warn if it's not reachable from
  links that were valid prior to a change
- create redirect rules automatically based on this information
---
 doc/manual/_redirects   | 30 ++++++++++++++++++++++++++++++
 doc/manual/redirects.js |  8 +++++---
 2 files changed, 35 insertions(+), 3 deletions(-)
 create mode 100644 doc/manual/_redirects

diff --git a/doc/manual/_redirects b/doc/manual/_redirects
new file mode 100644
index 000000000..4ea289d86
--- /dev/null
+++ b/doc/manual/_redirects
@@ -0,0 +1,30 @@
+# redirect rules for paths (server-side) to prevent link rot.
+# see ./redirects.js for redirects based on URL fragments (client-side)
+#
+# concrete user story this supports:
+# - user finds URL to the manual for Nix x.y
+# - Nix x.z (z > y) is the most recent release
+# - updating the version in the URL will show the right thing
+#
+# format documentation:
+# - https://docs.netlify.com/routing/redirects/#syntax-for-the-redirects-file
+# - https://docs.netlify.com/routing/redirects/redirect-options/
+#
+# conventions:
+# - always force (<CODE>!) since this allows re-using file names
+# - group related paths to ease readability
+# - always append new redirects to the end of the file
+# - redirects that should have been there but are missing can be inserted where they belong
+
+/expressions/expression-language /language/ 301!
+/expressions/language-values /language/values 301!
+/expressions/language-constructs /language/constructs 301!
+/expressions/language-operators /language/operators 301!
+/expressions/* /language/:splat 301!
+
+/package-management/basic-package-mgmt /command-ref/nix-env 301!
+
+/package-management/channels* /command-ref/nix-channel 301!
+
+/package-management/s3-substituter* /command-ref/new-cli/nix3-help-stores#s3-binary-cache-store 301!
+
diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js
index d1b10109d..3b507adf3 100644
--- a/doc/manual/redirects.js
+++ b/doc/manual/redirects.js
@@ -1,7 +1,9 @@
-// redirect rules for anchors ensure backwards compatibility of URLs.
-// this must be done on the client side, as web servers do not see the anchor part of the URL.
+// redirect rules for URL fragments (client-side) to prevent link rot.
+// this must be done on the client side, as web servers do not see the fragment part of the URL.
+// it will only work with JavaScript enabled in the browser, but this is the best we can do here.
+// see ./_redirects for path redirects (client-side)
 
-// redirections are declared as follows:
+// redirects are declared as follows:
 // each entry has as its key a path matching the requested URL path, relative to the mdBook document root.
 //
 //     IMPORTANT: it must specify the full path with file name and suffix

From f56401a114cb5504c3d74b893ce270ed28fd03e3 Mon Sep 17 00:00:00 2001
From: Moritz Angermann <moritz.angermann@gmail.com>
Date: Sat, 25 Nov 2023 11:26:57 +0800
Subject: [PATCH 092/421] `nix flake update` add deprecation warnings.

This builds on #8817, to add additional UX help for people with existing
muscle memory (or shell history) with --update-input and tries to gently
guide them towards the newly evolved CLI UI.

Co-authored-by: Cole Helbling <cole.e.helbling@outlook.com>
---
 src/libcmd/installables.cc | 24 ++++++++++++++++++++++++
 src/nix/flake.cc           |  8 +++++++-
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 1c6103020..68287b445 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -47,6 +47,16 @@ MixFlakeOptions::MixFlakeOptions()
 {
     auto category = "Common flake-related options";
 
+    addFlag({
+        .longName = "recreate-lock-file",
+        .description = "Recreate the flake's lock file from scratch.",
+        .category = category,
+        .handler = {[&]() {
+            lockFlags.recreateLockFile = true;
+            warn("'--recreate-lock-file' is deprecated and will be removed in a future version; use 'nix flake update' instead.");
+        }}
+    });
+
     addFlag({
         .longName = "no-update-lock-file",
         .description = "Do not allow any updates to the flake's lock file.",
@@ -79,6 +89,20 @@ MixFlakeOptions::MixFlakeOptions()
         .handler = {&lockFlags.commitLockFile, true}
     });
 
+    addFlag({
+        .longName = "update-input",
+        .description = "Update a specific flake input (ignoring its previous entry in the lock file).",
+        .category = category,
+        .labels = {"input-path"},
+        .handler = {[&](std::string s) {
+            warn("'--update-input' is a deprecated alias for 'flake update' and will be removed in a future version.");
+            lockFlags.inputUpdates.insert(flake::parseInputPath(s));
+        }},
+        .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
+            completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
+        }}
+    });
+
     addFlag({
         .longName = "override-input",
         .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.",
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 38938f09e..e0c67fdfa 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -89,7 +89,13 @@ public:
             .label="inputs",
             .optional=true,
             .handler={[&](std::string inputToUpdate){
-                auto inputPath = flake::parseInputPath(inputToUpdate);
+                InputPath inputPath;
+                try {
+                    inputPath = flake::parseInputPath(inputToUpdate);
+                } catch (Error & e) {
+                    warn("Invalid flake input '%s'. To update a specific flake, use 'nix flake update --flake %s' instead.", inputToUpdate, inputToUpdate);
+                    throw e;
+                }
                 if (lockFlags.inputUpdates.contains(inputPath))
                     warn("Input '%s' was specified multiple times. You may have done this by accident.");
                 lockFlags.inputUpdates.insert(inputPath);

From 75134b7513eb781074969fc8d6d865cc95063444 Mon Sep 17 00:00:00 2001
From: Sergei Trofimovich <slyich@gmail.com>
Date: Mon, 27 Nov 2023 08:56:24 +0000
Subject: [PATCH 093/421] libexpr: add missing dependency on
 'flake/call-flake.nix.gen.hh'

Without the change build for `eval.o` fails occasionally as:

    $ make src/libexpr/eval.o
      GEN    Makefile.config
      GEN    src/libexpr/primops/derivation.nix.gen.hh
      GEN    src/libexpr/fetchurl.nix.gen.hh
      GEN    src/libexpr/parser-tab.cc
      GEN    src/libexpr/lexer-tab.cc
    src/libexpr/lexer.l:314: warning, -s option given but default rule can be matched
      CXX    src/libexpr/eval.o
    src/libexpr/eval.cc:519:18: fatal error: flake/call-flake.nix.gen.hh: No such file or directory
      519 |         #include "flake/call-flake.nix.gen.hh"
          |                  ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    compilation terminated.
    make: *** [mk/patterns.mk:3: src/libexpr/eval.o] Error 1

Noticed in https://github.com/NixOS/nixpkgs/pull/269439
---
 src/libexpr/local.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index ed7bf9490..637f998b6 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -47,6 +47,6 @@ $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
 
 $(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh
 
-$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh
+$(d)/eval.o: $(d)/flake/call-flake.nix.gen.hh
 
 src/libexpr/primops/fromTOML.o:	ERROR_SWITCH_ENUM =

From e986d20bedfc054663632255388bbd33fec99114 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <7226587+thufschmitt@users.noreply.github.com>
Date: Mon, 27 Nov 2023 10:25:18 +0100
Subject: [PATCH 094/421] Remove an obsolete comment

---
 flake.nix | 2 --
 1 file changed, 2 deletions(-)

diff --git a/flake.nix b/flake.nix
index 1f7e7b2af..887ef93b4 100644
--- a/flake.nix
+++ b/flake.nix
@@ -500,8 +500,6 @@
           });
 
           boost-nix = final.boost.override {
-            # enableIcu arg is not yet supported
-            # but will be with next nixpkgs update
             enableIcu = false;
           };
 

From 384ffb4443fd47d04f36c6bcc6ebf476274673ab Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Mon, 27 Nov 2023 09:18:56 +0100
Subject: [PATCH 095/421] add deprecation warnings in documentation

this is hacky, but can serve as a stopgap until we can do it
programmatically.
---
 src/libcmd/installables.cc | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 68287b445..6e670efea 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -49,7 +49,13 @@ MixFlakeOptions::MixFlakeOptions()
 
     addFlag({
         .longName = "recreate-lock-file",
-        .description = "Recreate the flake's lock file from scratch.",
+        .description = R"(
+    Recreate the flake's lock file from scratch.
+
+    > **DEPRECATED**
+    >
+    > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead.
+        )",
         .category = category,
         .handler = {[&]() {
             lockFlags.recreateLockFile = true;
@@ -73,8 +79,13 @@ MixFlakeOptions::MixFlakeOptions()
 
     addFlag({
         .longName = "no-registries",
-        .description =
-          "Don't allow lookups in the flake registries. This option is deprecated; use `--no-use-registries`.",
+        .description = R"(
+    Don't allow lookups in the flake registries.
+
+    > **DEPRECATED**
+    >
+    > Use [`--no-use-registries`](#opt-no-use-registries) instead.
+        )",
         .category = category,
         .handler = {[&]() {
             lockFlags.useRegistries = false;
@@ -91,7 +102,13 @@ MixFlakeOptions::MixFlakeOptions()
 
     addFlag({
         .longName = "update-input",
-        .description = "Update a specific flake input (ignoring its previous entry in the lock file).",
+        .description = R"(
+    Update a specific flake input (ignoring its previous entry in the lock file).
+
+    > **DEPRECATED**
+    >
+    > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead.
+        )",
         .category = category,
         .labels = {"input-path"},
         .handler = {[&](std::string s) {

From f7bfec2806708573798d610cda101f27a24d9218 Mon Sep 17 00:00:00 2001
From: Robert Hensing <roberth@users.noreply.github.com>
Date: Mon, 27 Nov 2023 15:18:29 +0100
Subject: [PATCH 096/421] maintainers/release-notes: Improve DATE check

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 maintainers/release-notes | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/maintainers/release-notes b/maintainers/release-notes
index 2e1be64a4..34cd85a56 100755
--- a/maintainers/release-notes
+++ b/maintainers/release-notes
@@ -134,7 +134,7 @@ if [[ ! -n "${DATE:-}" ]]; then
 fi
 
 case "$DATE" in
-  [0-9]*-[0-9]*-[0-9]*)
+  [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9])
     ;;
   *)
     die "DATE must be YYYY-MM-DD, e.g. 2021-12-31 (DATE was set to $DATE)"

From 68c48756fece5aee77f9b44607afa9248d75e67c Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 27 Nov 2023 15:50:45 +0100
Subject: [PATCH 097/421] libexpr/local.mk: Make eval compile deps regular

Dependency is now entirely through the eval.cc rule.
All gen.hh deps are now there.
---
 src/libexpr/local.mk | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index 637f998b6..b37fe6f1d 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -45,8 +45,6 @@ $(foreach i, $(wildcard src/libexpr/flake/*.hh), \
 
 $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
 
-$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh
-
-$(d)/eval.o: $(d)/flake/call-flake.nix.gen.hh
+$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/call-flake.nix.gen.hh
 
 src/libexpr/primops/fromTOML.o:	ERROR_SWITCH_ENUM =

From 20cd5eb2b3668f5b95b6f020e1c258011c18ea33 Mon Sep 17 00:00:00 2001
From: Alois Wohlschlager <alois1@gmx-topmail.de>
Date: Mon, 27 Nov 2023 19:12:15 +0100
Subject: [PATCH 098/421] nix repl: Only hide the progress bar while waiting
 for user input

In commit 0d2163c6dcf03463fa91ec6d0d96c928ad907366, the progress bar was hidden
in nix repl because of a regression that caused it to interfere with user
input. Several users like(d) seeing the progress bar in the repl during builds.
Only hiding it while waiting for user input gives us the best of both worlds,
so do just that.
---
 src/libcmd/repl.cc | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc
index bf5643a5c..0986296ad 100644
--- a/src/libcmd/repl.cc
+++ b/src/libcmd/repl.cc
@@ -43,7 +43,6 @@ extern "C" {
 #include "finally.hh"
 #include "markdown.hh"
 #include "local-fs-store.hh"
-#include "progress-bar.hh"
 #include "print.hh"
 
 #if HAVE_BOEHMGC
@@ -262,13 +261,11 @@ void NixRepl::mainLoop()
     rl_set_list_possib_func(listPossibleCallback);
 #endif
 
-    /* Stop the progress bar because it interferes with the display of
-       the repl. */
-    stopProgressBar();
-
     std::string input;
 
     while (true) {
+        // Hide the progress bar while waiting for user input, so that it won't interfere.
+        logger->pause();
         // When continuing input from previous lines, don't print a prompt, just align to the same
         // number of chars as the prompt.
         if (!getLine(input, input.empty() ? "nix-repl> " : "          ")) {
@@ -278,6 +275,7 @@ void NixRepl::mainLoop()
             logger->cout("");
             break;
         }
+        logger->resume();
         try {
             if (!removeWhitespace(input).empty() && !processLine(input)) return;
         } catch (ParseError & e) {

From f300e11b056dea414d7d77bbc6e5a7dc5d9ddd41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Mon, 27 Nov 2023 19:41:30 +0100
Subject: [PATCH 099/421] Rename `nix show-config` to `nix config show`

Part of #7672
---
 doc/manual/local.mk                       |  4 ++--
 doc/manual/rl-next/nix-config-show.md     |  8 +++++++
 src/nix/{show-config.cc => config.cc}     | 27 ++++++++++++++++++++---
 src/nix/main.cc                           |  1 +
 tests/functional/config.sh                | 12 +++++-----
 tests/functional/experimental-features.sh |  8 +++----
 6 files changed, 45 insertions(+), 15 deletions(-)
 create mode 100644 doc/manual/rl-next/nix-config-show.md
 rename src/nix/{show-config.cc => config.cc} (67%)

diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index f22dfa69e..d568681d4 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -24,7 +24,7 @@ man-pages += $(foreach subcommand, \
 clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8
 
 # Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox.
-# Set cores to 0 because otherwise nix show-config resolves the cores based on the current machine
+# Set cores to 0 because otherwise `nix config show` resolves the cores based on the current machine
 dummy-env = env -i \
 	HOME=/dummy \
 	NIX_CONF_DIR=/dummy \
@@ -111,7 +111,7 @@ $(d)/nix.json: $(bindir)/nix
 	@mv $@.tmp $@
 
 $(d)/conf-file.json: $(bindir)/nix
-	$(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp
+	$(trace-gen) $(dummy-env) $(bindir)/nix config show --json --experimental-features nix-command > $@.tmp
 	@mv $@.tmp $@
 
 $(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix
diff --git a/doc/manual/rl-next/nix-config-show.md b/doc/manual/rl-next/nix-config-show.md
new file mode 100644
index 000000000..08ad207cb
--- /dev/null
+++ b/doc/manual/rl-next/nix-config-show.md
@@ -0,0 +1,8 @@
+synopsis: `nix config show`
+issues: #7672
+prs: #9477
+description: {
+
+`nix show-config` was renamed to `nix config show` to be more consistent with the rest of the command-line interface.
+
+}
diff --git a/src/nix/show-config.cc b/src/nix/config.cc
similarity index 67%
rename from src/nix/show-config.cc
rename to src/nix/config.cc
index 3530584f9..5b280d11d 100644
--- a/src/nix/show-config.cc
+++ b/src/nix/config.cc
@@ -7,11 +7,31 @@
 
 using namespace nix;
 
-struct CmdShowConfig : Command, MixJSON
+struct CmdConfig : virtual NixMultiCommand
+{
+    CmdConfig() : MultiCommand(RegisterCommand::getCommandsFor({"config"}))
+    { }
+
+    std::string description() override
+    {
+        return "manipulate the Nix configuration";
+    }
+
+    Category category() override { return catUtility; }
+
+    void run() override
+    {
+        if (!command)
+            throw UsageError("'nix config' requires a sub-command.");
+        command->second->run();
+    }
+};
+
+struct CmdConfigShow : Command, MixJSON
 {
     std::optional<std::string> name;
 
-    CmdShowConfig() {
+    CmdConfigShow() {
         expectArgs({
             .label = {"name"},
             .optional = true,
@@ -56,4 +76,5 @@ struct CmdShowConfig : Command, MixJSON
     }
 };
 
-static auto rShowConfig = registerCommand<CmdShowConfig>("show-config");
+static auto rCmdConfig = registerCommand<CmdConfig>("config");
+static auto rShowConfig = registerCommand2<CmdConfigShow>({"config", "show"});
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 73641f6d2..2a6c2f478 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -134,6 +134,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
         {"ping-store", {"store", "ping"}},
         {"sign-paths", {"store", "sign"}},
         {"show-derivation", {"derivation", "show"}},
+        {"show-config", {"config", "show"}},
         {"to-base16", {"hash", "to-base16"}},
         {"to-base32", {"hash", "to-base32"}},
         {"to-base64", {"hash", "to-base64"}},
diff --git a/tests/functional/config.sh b/tests/functional/config.sh
index 0780c55d0..324fe95bd 100644
--- a/tests/functional/config.sh
+++ b/tests/functional/config.sh
@@ -40,20 +40,20 @@ files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | x
 # Test that it's possible to load the config from a custom location
 here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")
 export NIX_USER_CONF_FILES=$here/config/nix-with-substituters.conf
-var=$(nix show-config | grep '^substituters =' | cut -d '=' -f 2 | xargs)
+var=$(nix config show | grep '^substituters =' | cut -d '=' -f 2 | xargs)
 [[ $var == https://example.com ]]
 
 # Test that it's possible to load config from the environment
-prev=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs)
+prev=$(nix config show | grep '^cores' | cut -d '=' -f 2 | xargs)
 export NIX_CONFIG="cores = 4242"$'\n'"experimental-features = nix-command flakes"
-exp_cores=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs)
-exp_features=$(nix show-config | grep '^experimental-features' | cut -d '=' -f 2 | xargs)
+exp_cores=$(nix config show | grep '^cores' | cut -d '=' -f 2 | xargs)
+exp_features=$(nix config show | grep '^experimental-features' | cut -d '=' -f 2 | xargs)
 [[ $prev != $exp_cores ]]
 [[ $exp_cores == "4242" ]]
 # flakes implies fetch-tree
 [[ $exp_features == "fetch-tree flakes nix-command" ]]
 
 # Test that it's possible to retrieve a single setting's value
-val=$(nix show-config | grep '^warn-dirty' | cut -d '=' -f  2 | xargs)
-val2=$(nix show-config warn-dirty)
+val=$(nix config show | grep '^warn-dirty' | cut -d '=' -f  2 | xargs)
+val2=$(nix config show warn-dirty)
 [[ $val == $val2 ]]
diff --git a/tests/functional/experimental-features.sh b/tests/functional/experimental-features.sh
index 607bf0a8e..9ee4a53d4 100644
--- a/tests/functional/experimental-features.sh
+++ b/tests/functional/experimental-features.sh
@@ -31,7 +31,7 @@ source common.sh
 NIX_CONFIG='
   experimental-features = nix-command
   accept-flake-config = true
-' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
+' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
 grepQuiet "false" $TEST_ROOT/stdout
 grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr
 
@@ -39,7 +39,7 @@ grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature '
 NIX_CONFIG='
   accept-flake-config = true
   experimental-features = nix-command
-' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
+' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
 grepQuiet "false" $TEST_ROOT/stdout
 grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr
 
@@ -47,7 +47,7 @@ grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature '
 NIX_CONFIG='
   experimental-features = nix-command flakes
   accept-flake-config = true
-' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
+' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
 grepQuiet "true" $TEST_ROOT/stdout
 grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr
 
@@ -55,7 +55,7 @@ grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr
 NIX_CONFIG='
   accept-flake-config = true
   experimental-features = nix-command flakes
-' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
+' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
 grepQuiet "true" $TEST_ROOT/stdout
 grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr
 

From 6d1605818c12461edc9f4ee17a7929fdc8fe916c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Mon, 27 Nov 2023 19:46:05 +0100
Subject: [PATCH 100/421] Rename `nix doctor` to `nix config check`

Fix #7672
---
 doc/manual/src/contributing/cli-guideline.md | 2 +-
 src/nix/{doctor.cc => config-check.cc}       | 4 ++--
 src/nix/main.cc                              | 1 +
 3 files changed, 4 insertions(+), 3 deletions(-)
 rename src/nix/{doctor.cc => config-check.cc} (97%)

diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/contributing/cli-guideline.md
index e53d2d178..8dbac45b0 100644
--- a/doc/manual/src/contributing/cli-guideline.md
+++ b/doc/manual/src/contributing/cli-guideline.md
@@ -87,7 +87,7 @@ impacted the most by bad user experience.
       and [aligning of text](#text-alignment).
     - [Autocomplete](#shell-completion) of options.
 
-  Examples of such commands: `nix doctor`, `nix edit`, `nix eval`, ...
+  Examples of such commands: `nix edit`, `nix eval`, ...
 
 - **Utility and scripting commands**
 
diff --git a/src/nix/doctor.cc b/src/nix/config-check.cc
similarity index 97%
rename from src/nix/doctor.cc
rename to src/nix/config-check.cc
index 59f9e3e5d..410feca2f 100644
--- a/src/nix/doctor.cc
+++ b/src/nix/config-check.cc
@@ -38,7 +38,7 @@ void checkInfo(const std::string & msg) {
 
 }
 
-struct CmdDoctor : StoreCommand
+struct CmdConfigCheck : StoreCommand
 {
     bool success = true;
 
@@ -152,4 +152,4 @@ struct CmdDoctor : StoreCommand
     }
 };
 
-static auto rCmdDoctor = registerCommand<CmdDoctor>("doctor");
+static auto rCmdConfigCheck = registerCommand2<CmdConfigCheck>({ "config", "check" });
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 2a6c2f478..d715d0400 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -139,6 +139,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
         {"to-base32", {"hash", "to-base32"}},
         {"to-base64", {"hash", "to-base64"}},
         {"verify", {"store", "verify"}},
+        {"doctor", {"config", "check"}},
     };
 
     bool aliasUsed = false;

From 52e0911302b20336c1600b60a98894423e110d7d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 25 Nov 2023 00:33:21 -0500
Subject: [PATCH 101/421] Use `buildprefix` in a few more places

`installcheck` doesn't yet work, but the rest of the build can now
happen mostly inside a separate build directory.

Progress on #9342

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 Makefile                               |  6 ++++--
 doc/manual/src/contributing/hacking.md | 25 +++++++++++++++++++++++++
 mk/build-dir.mk                        | 10 ++++++++++
 mk/install-dirs.mk                     | 11 +++++++++++
 mk/lib.mk                              | 25 +++----------------------
 mk/templates.mk                        |  8 ++++----
 src/libcmd/local.mk                    |  2 +-
 src/libexpr/local.mk                   |  4 ++--
 src/libmain/local.mk                   |  2 +-
 src/libstore/local.mk                  |  2 +-
 tests/functional/local.mk              |  4 ++--
 11 files changed, 64 insertions(+), 35 deletions(-)
 create mode 100644 mk/build-dir.mk
 create mode 100644 mk/install-dirs.mk

diff --git a/Makefile b/Makefile
index 4f4ac0c6e..77974074c 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,7 @@
--include Makefile.config
-clean-files += Makefile.config
+include mk/build-dir.mk
+
+-include $(buildprefix)Makefile.config
+clean-files += $(buildprefix)Makefile.config
 
 ifeq ($(ENABLE_BUILD), yes)
 makefiles = \
diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index 3291d5a20..0a95334f7 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -146,6 +146,31 @@ $ nix build .#packages.aarch64-linux.default
 Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`).
 Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms.
 
+### Building for multiple platforms at once
+
+It is useful to perform multiple cross and native builds on the same source tree,
+for example to ensure that better support for one platform doesn't break the build for another.
+In order to facilitate this, Nix has some support for being built out of tree – that is, placing build artefacts in a different directory than the source code:
+
+1. Create a directory for the build, e.g.
+
+   ```bash
+   mkdir build
+   ```
+
+2. Run the configure script from that directory, e.g.
+
+   ```bash
+   cd build
+   ../configure <configure flags>
+   ```
+
+3. Run make from the source directory, but with the build directory specified, e.g.
+
+   ```bash
+   make builddir=build <make flags>
+   ```
+
 ## System type
 
 Nix uses a string with he following format to identify the *system type* or *platform* it runs on:
diff --git a/mk/build-dir.mk b/mk/build-dir.mk
new file mode 100644
index 000000000..02f4cae60
--- /dev/null
+++ b/mk/build-dir.mk
@@ -0,0 +1,10 @@
+# Initialise support for build directories.
+builddir ?=
+
+ifdef builddir
+  buildprefix = $(builddir)/
+  buildprefixrel = $(builddir)
+else
+  buildprefix =
+  buildprefixrel = .
+endif
diff --git a/mk/install-dirs.mk b/mk/install-dirs.mk
new file mode 100644
index 000000000..732b0d6fc
--- /dev/null
+++ b/mk/install-dirs.mk
@@ -0,0 +1,11 @@
+# Default installation paths.
+prefix ?= /usr/local
+libdir ?= $(prefix)/lib
+bindir ?= $(prefix)/bin
+libexecdir ?= $(prefix)/libexec
+datadir ?= $(prefix)/share
+localstatedir ?= $(prefix)/var
+sysconfdir ?= $(prefix)/etc
+mandir ?= $(prefix)/share/man
+
+DESTDIR ?=
diff --git a/mk/lib.mk b/mk/lib.mk
index 49abe9862..3d503364f 100644
--- a/mk/lib.mk
+++ b/mk/lib.mk
@@ -43,27 +43,6 @@ define newline
 endef
 
 
-# Default installation paths.
-prefix ?= /usr/local
-libdir ?= $(prefix)/lib
-bindir ?= $(prefix)/bin
-libexecdir ?= $(prefix)/libexec
-datadir ?= $(prefix)/share
-localstatedir ?= $(prefix)/var
-sysconfdir ?= $(prefix)/etc
-mandir ?= $(prefix)/share/man
-
-
-# Initialise support for build directories.
-builddir ?=
-
-ifdef builddir
-  buildprefix = $(builddir)/
-else
-  buildprefix =
-endif
-
-
 # Pass -fPIC if we're building dynamic libraries.
 BUILD_SHARED_LIBS ?= 1
 
@@ -94,6 +73,8 @@ ifeq ($(BUILD_DEBUG), 1)
 endif
 
 
+include mk/build-dir.mk
+include mk/install-dirs.mk
 include mk/functions.mk
 include mk/tracing.mk
 include mk/clean.mk
@@ -112,7 +93,7 @@ define include-sub-makefile
   include $(1)
 endef
 
-$(foreach mf, $(makefiles), $(eval $(call include-sub-makefile, $(mf))))
+$(foreach mf, $(makefiles), $(eval $(call include-sub-makefile,$(mf))))
 
 
 # Instantiate stuff.
diff --git a/mk/templates.mk b/mk/templates.mk
index c7ac7afbf..866bdc17f 100644
--- a/mk/templates.mk
+++ b/mk/templates.mk
@@ -10,10 +10,10 @@ endef
 
 ifneq ($(MAKECMDGOALS), clean)
 
-%.h: %.h.in
-	$(trace-gen) rm -f $@ && ./config.status --quiet --header=$@
+$(buildprefix)%.h: %.h.in
+	$(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --header=$(@:$(buildprefix)%=%)
 
-%: %.in
-	$(trace-gen) rm -f $@ && ./config.status --quiet --file=$@
+$(buildprefix)%: %.in
+	$(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --file=$(@:$(buildprefix)%=%)
 
 endif
diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk
index 541a7d2ba..afd35af08 100644
--- a/src/libcmd/local.mk
+++ b/src/libcmd/local.mk
@@ -12,4 +12,4 @@ libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread
 
 libcmd_LIBS = libstore libutil libexpr libmain libfetchers
 
-$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644))
+$(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644))
diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index 18d1bc95c..c07a18bb5 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -36,7 +36,7 @@ $(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l
 
 clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
 
-$(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644))
+$(eval $(call install-file-in, $(buildprefix)$(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644))
 
 $(foreach i, $(wildcard src/libexpr/value/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644)))
@@ -47,4 +47,4 @@ $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
 
 $(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/call-flake.nix.gen.hh
 
-src/libexpr/primops/fromTOML.o:	ERROR_SWITCH_ENUM =
+$(buildprefix)src/libexpr/primops/fromTOML.o:	ERROR_SWITCH_ENUM =
diff --git a/src/libmain/local.mk b/src/libmain/local.mk
index 99da95e27..5c7061863 100644
--- a/src/libmain/local.mk
+++ b/src/libmain/local.mk
@@ -14,4 +14,4 @@ libmain_LIBS = libstore libutil
 
 libmain_ALLOW_UNDEFINED = 1
 
-$(eval $(call install-file-in, $(d)/nix-main.pc, $(libdir)/pkgconfig, 0644))
+$(eval $(call install-file-in, $(buildprefix)$(d)/nix-main.pc, $(libdir)/pkgconfig, 0644))
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index 0be0bf310..68ccdc409 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -59,7 +59,7 @@ $(d)/build.cc:
 
 clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
 
-$(eval $(call install-file-in, $(d)/nix-store.pc, $(libdir)/pkgconfig, 0644))
+$(eval $(call install-file-in, $(buildprefix)$(d)/nix-store.pc, $(libdir)/pkgconfig, 0644))
 
 $(foreach i, $(wildcard src/libstore/builtins/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644)))
diff --git a/tests/functional/local.mk b/tests/functional/local.mk
index 8d584142a..10b399d75 100644
--- a/tests/functional/local.mk
+++ b/tests/functional/local.mk
@@ -140,9 +140,9 @@ ifeq ($(ENABLE_BUILD), yes)
 endif
 
 $(d)/test-libstoreconsumer.sh.test $(d)/test-libstoreconsumer.sh.test-debug: \
-  $(d)/test-libstoreconsumer/test-libstoreconsumer
+  $(buildprefix)$(d)/test-libstoreconsumer/test-libstoreconsumer
 $(d)/plugins.sh.test $(d)/plugins.sh.test-debug: \
-  $(d)/plugins/libplugintest.$(SO_EXT)
+  $(buildprefix)$(d)/plugins/libplugintest.$(SO_EXT)
 
 install-tests += $(foreach x, $(nix_tests), $(d)/$(x))
 

From 02bd821f2e71372d31bbe6700bd68086cc2ee70a Mon Sep 17 00:00:00 2001
From: Alex Ameen <alex.ameen.tx@gmail.com>
Date: Wed, 29 Nov 2023 19:26:39 -0600
Subject: [PATCH 102/421] fix: `nlohmann::adl_serializer` for `std::optional`
 (#9147)

This allows templates such as `NLOHMANN_DEFINE_TYPE_*` templates and other generators with things like `std::vector<std::optional<T>>`.

Co-authored-by: John Ericson <John.Ericson@Obsidian.Systems>
---
 src/libutil/json-utils.hh       | 19 ++++++++---
 src/libutil/tests/json-utils.cc | 58 +++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+), 5 deletions(-)
 create mode 100644 src/libutil/tests/json-utils.cc

diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh
index 77c63595c..06dd80cf7 100644
--- a/src/libutil/json-utils.hh
+++ b/src/libutil/json-utils.hh
@@ -78,20 +78,29 @@ namespace nlohmann {
  */
 template<typename T>
 struct adl_serializer<std::optional<T>> {
-    static std::optional<T> from_json(const json & json) {
+    /**
+     * @brief Convert a JSON type to an `optional<T>` treating
+     *        `null` as `std::nullopt`.
+     */
+    static void from_json(const json & json, std::optional<T> & t) {
         static_assert(
             nix::json_avoids_null<T>::value,
             "null is already in use for underlying type's JSON");
-        return json.is_null()
+        t = json.is_null()
             ? std::nullopt
-            : std::optional { adl_serializer<T>::from_json(json) };
+            : std::make_optional(json.template get<T>());
     }
-    static void to_json(json & json, std::optional<T> t) {
+
+    /**
+     *  @brief Convert an optional type to a JSON type  treating `std::nullopt`
+     *         as `null`.
+     */
+    static void to_json(json & json, const std::optional<T> & t) {
         static_assert(
             nix::json_avoids_null<T>::value,
             "null is already in use for underlying type's JSON");
         if (t)
-            adl_serializer<T>::to_json(json, *t);
+            json = *t;
         else
             json = nullptr;
     }
diff --git a/src/libutil/tests/json-utils.cc b/src/libutil/tests/json-utils.cc
new file mode 100644
index 000000000..f0ce15c93
--- /dev/null
+++ b/src/libutil/tests/json-utils.cc
@@ -0,0 +1,58 @@
+#include <vector>
+#include <optional>
+
+#include <gtest/gtest.h>
+
+#include "json-utils.hh"
+
+namespace nix {
+
+/* Test `to_json` and `from_json` with `std::optional` types.
+ * We are specifically interested in whether we can _nest_ optionals in STL
+ * containers so we that we can leverage existing adl_serializer templates. */
+
+TEST(to_json, optionalInt) {
+    std::optional<int> val = std::make_optional(420);
+    ASSERT_EQ(nlohmann::json(val), nlohmann::json(420));
+    val = std::nullopt;
+    ASSERT_EQ(nlohmann::json(val), nlohmann::json(nullptr));
+}
+
+TEST(to_json, vectorOfOptionalInts) {
+    std::vector<std::optional<int>> vals = {
+        std::make_optional(420),
+        std::nullopt,
+    };
+    ASSERT_EQ(nlohmann::json(vals), nlohmann::json::parse("[420,null]"));
+}
+
+TEST(to_json, optionalVectorOfInts) {
+    std::optional<std::vector<int>> val = std::make_optional(std::vector<int> {
+        -420,
+        420,
+    });
+    ASSERT_EQ(nlohmann::json(val), nlohmann::json::parse("[-420,420]"));
+    val = std::nullopt;
+    ASSERT_EQ(nlohmann::json(val), nlohmann::json(nullptr));
+}
+
+TEST(from_json, optionalInt) {
+    nlohmann::json json = 420;
+    std::optional<int> val = json;
+    ASSERT_TRUE(val.has_value());
+    ASSERT_EQ(*val, 420);
+    json = nullptr;
+    json.get_to(val);
+    ASSERT_FALSE(val.has_value());
+}
+
+TEST(from_json, vectorOfOptionalInts) {
+    nlohmann::json json = { 420, nullptr };
+    std::vector<std::optional<int>> vals = json;
+    ASSERT_EQ(vals.size(), 2);
+    ASSERT_TRUE(vals.at(0).has_value());
+    ASSERT_EQ(*vals.at(0), 420);
+    ASSERT_FALSE(vals.at(1).has_value());
+}
+
+} /* namespace nix */

From a7115a47ef0d83ea81b494f6bc5b11d8286e0672 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 29 Nov 2023 21:03:56 -0500
Subject: [PATCH 103/421] Improve ACL clearing support (fixing FreeBSD build)

The problem was that f880469173061a07f0b2a24734932c5a9ad633c6 forgot
that the `#include <sys/xattr.h>` was guarded by an `#ifdef __linux__`.

However, the build failure was only on FreeBSD --- turns out other
platforms have this header too!

The fix therefore uses a new configure check so we properly clear ACLs
on more platforms.
---
 configure.ac                          | 2 ++
 src/libstore/local-store.cc           | 1 -
 src/libstore/posix-fs-canonicalise.cc | 6 ++++--
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/configure.ac b/configure.ac
index 71e93feaa..f8b937eb5 100644
--- a/configure.ac
+++ b/configure.ac
@@ -282,6 +282,8 @@ case "$host_os" in
 esac
 AC_SUBST(HAVE_SECCOMP, [$have_seccomp])
 
+# Optional dependencies for better normalizing file system data
+AC_CHECK_HEADERS[sys/xattr.h]
 
 # Look for aws-cpp-sdk-s3.
 AC_LANG_PUSH(C++)
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 4ff75f528..9ed061b01 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -34,7 +34,6 @@
 #include <sys/statvfs.h>
 #include <sys/mount.h>
 #include <sys/ioctl.h>
-#include <sys/xattr.h>
 #endif
 
 #ifdef __CYGWIN__
diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc
index cc3ab0b74..f38fa8369 100644
--- a/src/libstore/posix-fs-canonicalise.cc
+++ b/src/libstore/posix-fs-canonicalise.cc
@@ -1,4 +1,6 @@
-#include <sys/xattr.h>
+#if HAVE_SYS_XATTR_H
+# include <sys/xattr.h>
+#endif
 
 #include "posix-fs-canonicalise.hh"
 #include "file-system.hh"
@@ -76,7 +78,7 @@ static void canonicalisePathMetaData_(
     if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
         throw Error("file '%1%' has an unsupported type", path);
 
-#if __linux__
+#ifdef HAVE_SYS_XATTR_H
     /* Remove extended attributes / ACLs. */
     ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0);
 

From d536c57e878a04f795c1ef8ee3232a47035da2cf Mon Sep 17 00:00:00 2001
From: Federico Pellegrin <fede@evolware.org>
Date: Tue, 17 Aug 2021 04:26:41 +0200
Subject: [PATCH 104/421] Docs build: depend on locally built nix executable
 and not installed one

Previously many of the documentation targets were depending on
`$(bindir)/nix` which is the installed version. This meant that its
install rules would be triggered (which in chain would also trigger the
install of libraries, as reported in #5140). Therefore a build of the
documentation without an installation would not be possible (which apart
from doing unwanted operations it may also generate permission problems
for example).

The fix makes the rules depend on `$(nix_PATH)` instead, which is the
executable in the build tree.
---
 Makefile            | 11 ++++++++---
 doc/manual/local.mk | 35 ++++++++++++++++++++---------------
 2 files changed, 28 insertions(+), 18 deletions(-)

diff --git a/Makefile b/Makefile
index 4f4ac0c6e..957920686 100644
--- a/Makefile
+++ b/Makefile
@@ -19,9 +19,7 @@ makefiles = \
   misc/zsh/local.mk \
   misc/systemd/local.mk \
   misc/launchd/local.mk \
-  misc/upstart/local.mk \
-  doc/manual/local.mk \
-  doc/internal-api/local.mk
+  misc/upstart/local.mk
 endif
 
 ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes)
@@ -55,4 +53,11 @@ endif
 
 include mk/lib.mk
 
+# Must be included after `mk/lib.mk` so rules refer to variables defined
+# by the library. Rules are not "lazy" like variables, unfortunately.
+ifeq ($(ENABLE_BUILD), yes)
+$(eval $(call include-sub-makefile, doc/manual/local.mk))
+$(eval $(call include-sub-makefile, doc/internal-api/local.mk))
+endif
+
 GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++2a -I src
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index d568681d4..fa9db9f02 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -1,5 +1,10 @@
 ifeq ($(doc_generate),yes)
 
+# The version of Nix used to generate the doc. Can also be
+# `$(nix_INSTALL_PATH)` or just `nix` (to grap ambient from the `PATH`),
+# if one prefers.
+doc_nix = $(nix_PATH)
+
 MANUAL_SRCS := \
 	$(call rwildcard, $(d)/src, *.md) \
 	$(call rwildcard, $(d)/src, */*.md)
@@ -32,7 +37,7 @@ dummy-env = env -i \
 	NIX_STATE_DIR=/dummy \
 	NIX_CONFIG='cores = 0'
 
-nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw
+nix-eval = $(dummy-env) $(doc_nix) eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw
 
 # re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution
 define process-includes
@@ -96,52 +101,52 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src
 	@cp $< $@
 	@$(call process-includes,$@,$@)
 
-$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(bindir)/nix
+$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(doc_nix)
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)'
 	@mv $@.tmp $@
 
-$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix
+$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(doc_nix)
 	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "conf"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
 	@mv $@.tmp $@
 
-$(d)/nix.json: $(bindir)/nix
-	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-cli > $@.tmp
+$(d)/nix.json: $(doc_nix)
+	$(trace-gen) $(dummy-env) $(doc_nix) __dump-cli > $@.tmp
 	@mv $@.tmp $@
 
-$(d)/conf-file.json: $(bindir)/nix
-	$(trace-gen) $(dummy-env) $(bindir)/nix config show --json --experimental-features nix-command > $@.tmp
+$(d)/conf-file.json: $(doc_nix)
+	$(trace-gen) $(dummy-env) $(doc_nix) config show --json --experimental-features nix-command > $@.tmp
 	@mv $@.tmp $@
 
-$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix
+$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix)
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))'
 	@mv $@.tmp $@
 
-$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(bindir)/nix
+$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(doc_nix)
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))'
 	@mv $@.tmp $@
 
-$(d)/xp-features.json: $(bindir)/nix
-	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-xp-features > $@.tmp
+$(d)/xp-features.json: $(doc_nix)
+	$(trace-gen) $(dummy-env) $(doc_nix) __dump-xp-features > $@.tmp
 	@mv $@.tmp $@
 
-$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix
+$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(doc_nix)
 	@cat doc/manual/src/language/builtins-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<)).builtins' >> $@.tmp;
 	@cat doc/manual/src/language/builtins-suffix.md >> $@.tmp
 	@mv $@.tmp $@
 
-$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(bindir)/nix
+$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(doc_nix)
 	@cat doc/manual/src/language/builtin-constants-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtin-constants.nix (builtins.fromJSON (builtins.readFile $<)).constants' >> $@.tmp;
 	@cat doc/manual/src/language/builtin-constants-suffix.md >> $@.tmp
 	@mv $@.tmp $@
 
-$(d)/language.json: $(bindir)/nix
-	$(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp
+$(d)/language.json: $(doc_nix)
+	$(trace-gen) $(dummy-env) $(doc_nix) __dump-language > $@.tmp
 	@mv $@.tmp $@
 
 # Generate "Upcoming release" notes (or clear it and remove from menu)

From 743232bf04d8ade18cfa2c791ed466ce48519878 Mon Sep 17 00:00:00 2001
From: Greg Pfeil <greg@technomadic.org>
Date: Thu, 30 Nov 2023 00:17:25 -0700
Subject: [PATCH 105/421] =?UTF-8?q?Don=E2=80=99t=20use=20`execvp`=20when?=
 =?UTF-8?q?=20we=20know=20the=20path?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/nix/develop.cc |  2 +-
 src/nix/fmt.cc     |  2 +-
 src/nix/run.cc     | 10 +++++++---
 src/nix/run.hh     |  1 +
 4 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 38482ed42..ae9be79a3 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -662,7 +662,7 @@ struct CmdDevelop : Common, MixEnvironment
             }
         }
 
-        runProgramInStore(store, shell, args, buildEnvironment.getSystem());
+        runProgramInStore(store, true, shell, args, buildEnvironment.getSystem());
     }
 };
 
diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc
index c85eacded..396c93dbb 100644
--- a/src/nix/fmt.cc
+++ b/src/nix/fmt.cc
@@ -49,7 +49,7 @@ struct CmdFmt : SourceExprCommand {
             }
         }
 
-        runProgramInStore(store, app.program, programArgs);
+        runProgramInStore(store, false, app.program, programArgs);
     };
 };
 
diff --git a/src/nix/run.cc b/src/nix/run.cc
index ea0a17897..d531f712d 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -25,6 +25,7 @@ std::string chrootHelperName = "__run_in_chroot";
 namespace nix {
 
 void runProgramInStore(ref<Store> store,
+    bool search,
     const std::string & program,
     const Strings & args,
     std::optional<std::string_view> system)
@@ -58,7 +59,10 @@ void runProgramInStore(ref<Store> store,
     if (system)
         setPersonality(*system);
 
-    execvp(program.c_str(), stringsToCharPtrs(args).data());
+    if (search)
+        execvp(program.c_str(), stringsToCharPtrs(args).data());
+    else
+        execv(program.c_str(), stringsToCharPtrs(args).data());
 
     throw SysError("unable to execute '%s'", program);
 }
@@ -132,7 +136,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment
         Strings args;
         for (auto & arg : command) args.push_back(arg);
 
-        runProgramInStore(store, *command.begin(), args);
+        runProgramInStore(store, true, *command.begin(), args);
     }
 };
 
@@ -194,7 +198,7 @@ struct CmdRun : InstallableValueCommand
         Strings allArgs{app.program};
         for (auto & i : args) allArgs.push_back(i);
 
-        runProgramInStore(store, app.program, allArgs);
+        runProgramInStore(store, false, app.program, allArgs);
     }
 };
 
diff --git a/src/nix/run.hh b/src/nix/run.hh
index 97ddef19b..c62287e7e 100644
--- a/src/nix/run.hh
+++ b/src/nix/run.hh
@@ -6,6 +6,7 @@
 namespace nix {
 
 void runProgramInStore(ref<Store> store,
+    bool search,
     const std::string & program,
     const Strings & args,
     std::optional<std::string_view> system = std::nullopt);

From f99e468640ab0eac7f07ac2b328222eb45dee8d8 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 27 Nov 2023 08:48:11 -0500
Subject: [PATCH 106/421] Avoid `<name>/<name>` in documentation URLs

They are redundant and look weird.
---
 doc/manual/_redirects                          | 18 ++++++++++++++----
 doc/manual/src/SUMMARY.md.in                   | 14 +++++++-------
 .../{advanced-topics.md => index.md}           |  0
 doc/manual/src/architecture/architecture.md    |  2 +-
 .../command-ref/{command-ref.md => index.md}   |  0
 doc/manual/src/contributing/hacking.md         |  2 +-
 .../contributing/{contributing.md => index.md} |  0
 .../installation/{installation.md => index.md} |  0
 .../{package-management.md => index.md}        |  0
 .../src/protocols/{protocols.md => index.md}   |  0
 doc/manual/src/quick-start.md                  |  2 +-
 .../{release-notes.md => index.md}             |  0
 12 files changed, 24 insertions(+), 14 deletions(-)
 rename doc/manual/src/advanced-topics/{advanced-topics.md => index.md} (100%)
 rename doc/manual/src/command-ref/{command-ref.md => index.md} (100%)
 rename doc/manual/src/contributing/{contributing.md => index.md} (100%)
 rename doc/manual/src/installation/{installation.md => index.md} (100%)
 rename doc/manual/src/package-management/{package-management.md => index.md} (100%)
 rename doc/manual/src/protocols/{protocols.md => index.md} (100%)
 rename doc/manual/src/release-notes/{release-notes.md => index.md} (100%)

diff --git a/doc/manual/_redirects b/doc/manual/_redirects
index 4ea289d86..2038671d7 100644
--- a/doc/manual/_redirects
+++ b/doc/manual/_redirects
@@ -13,18 +13,28 @@
 # conventions:
 # - always force (<CODE>!) since this allows re-using file names
 # - group related paths to ease readability
-# - always append new redirects to the end of the file
+# - keep in alphabetical/wildcards-last order, which will reduce version control conflicts
 # - redirects that should have been there but are missing can be inserted where they belong
 
+/advanced-topics/advanced-topics /advanced-topics 301!
+
+/command-ref/command-ref /command-ref 301!
+
+/contributing/contributing /contributing 301!
+
 /expressions/expression-language /language/ 301!
-/expressions/language-values /language/values 301!
 /expressions/language-constructs /language/constructs 301!
 /expressions/language-operators /language/operators 301!
+/expressions/language-values /language/values 301!
 /expressions/* /language/:splat 301!
 
+/installation/installation /installation 301!
+
 /package-management/basic-package-mgmt /command-ref/nix-env 301!
-
 /package-management/channels* /command-ref/nix-channel 301!
-
+/package-management/package-management /package-management 301!
 /package-management/s3-substituter* /command-ref/new-cli/nix3-help-stores#s3-binary-cache-store 301!
 
+/protocols/protocols /protocols 301!
+
+/release-notes/release-notes /release-notes 301!
diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in
index 8e7b4eeab..686d3e8d7 100644
--- a/doc/manual/src/SUMMARY.md.in
+++ b/doc/manual/src/SUMMARY.md.in
@@ -2,7 +2,7 @@
 
 - [Introduction](introduction.md)
 - [Quick Start](quick-start.md)
-- [Installation](installation/installation.md)
+- [Installation](installation/index.md)
   - [Supported Platforms](installation/supported-platforms.md)
   - [Installing a Binary Distribution](installation/installing-binary.md)
   - [Installing Nix from Source](installation/installing-source.md)
@@ -31,11 +31,11 @@
     - [Import From Derivation](language/import-from-derivation.md)
   - [Built-in Constants](language/builtin-constants.md)
   - [Built-in Functions](language/builtins.md)
-- [Package Management](package-management/package-management.md)
+- [Package Management](package-management/index.md)
   - [Profiles](package-management/profiles.md)
   - [Garbage Collection](package-management/garbage-collection.md)
     - [Garbage Collector Roots](package-management/garbage-collector-roots.md)
-- [Advanced Topics](advanced-topics/advanced-topics.md)
+- [Advanced Topics](advanced-topics/index.md)
   - [Sharing Packages Between Machines](package-management/sharing-packages.md)
     - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md)
     - [Copying Closures via SSH](package-management/copy-closure.md)
@@ -45,7 +45,7 @@
   - [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md)
   - [Verifying Build Reproducibility](advanced-topics/diff-hook.md)
   - [Using the `post-build-hook`](advanced-topics/post-build-hook.md)
-- [Command Reference](command-ref/command-ref.md)
+- [Command Reference](command-ref/index.md)
   - [Common Options](command-ref/opt-common.md)
   - [Common Environment Variables](command-ref/env-common.md)
   - [Main Commands](command-ref/main-commands.md)
@@ -102,18 +102,18 @@
     - [Channels](command-ref/files/channels.md)
     - [Default Nix expression](command-ref/files/default-nix-expression.md)
 - [Architecture and Design](architecture/architecture.md)
-- [Protocols](protocols/protocols.md)
+- [Protocols](protocols/index.md)
   - [Serving Tarball Flakes](protocols/tarball-fetcher.md)
   - [Derivation "ATerm" file format](protocols/derivation-aterm.md)
 - [Glossary](glossary.md)
-- [Contributing](contributing/contributing.md)
+- [Contributing](contributing/index.md)
   - [Hacking](contributing/hacking.md)
   - [Testing](contributing/testing.md)
   - [Documentation](contributing/documentation.md)
   - [Experimental Features](contributing/experimental-features.md)
   - [CLI guideline](contributing/cli-guideline.md)
   - [C++ style guide](contributing/cxx.md)
-- [Release Notes](release-notes/release-notes.md)
+- [Release Notes](release-notes/index.md)
 {{#include ./SUMMARY-rl-next.md}}
   - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md)
   - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md)
diff --git a/doc/manual/src/advanced-topics/advanced-topics.md b/doc/manual/src/advanced-topics/index.md
similarity index 100%
rename from doc/manual/src/advanced-topics/advanced-topics.md
rename to doc/manual/src/advanced-topics/index.md
diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md
index 79429508f..2fec4ed20 100644
--- a/doc/manual/src/architecture/architecture.md
+++ b/doc/manual/src/architecture/architecture.md
@@ -52,7 +52,7 @@ The following [concept map] shows its main components (rectangles), the objects
                                             '---------------'
 ```
 
-At the top is the [command line interface](../command-ref/command-ref.md) that drives the underlying layers.
+At the top is the [command line interface](../command-ref/index.md) that drives the underlying layers.
 
 The [Nix language](../language/index.md) evaluator transforms Nix expressions into self-contained *build plans*, which are used to derive *build results* from referenced *build inputs*.
 
diff --git a/doc/manual/src/command-ref/command-ref.md b/doc/manual/src/command-ref/index.md
similarity index 100%
rename from doc/manual/src/command-ref/command-ref.md
rename to doc/manual/src/command-ref/index.md
diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index 0a95334f7..9de5ad39b 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -10,7 +10,7 @@ $ cd nix
 
 The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions].
 
-[installation instructions]: ../installation/installation.md
+[installation instructions]: ../installation/index.md
 
 ## Building Nix with flakes
 
diff --git a/doc/manual/src/contributing/contributing.md b/doc/manual/src/contributing/index.md
similarity index 100%
rename from doc/manual/src/contributing/contributing.md
rename to doc/manual/src/contributing/index.md
diff --git a/doc/manual/src/installation/installation.md b/doc/manual/src/installation/index.md
similarity index 100%
rename from doc/manual/src/installation/installation.md
rename to doc/manual/src/installation/index.md
diff --git a/doc/manual/src/package-management/package-management.md b/doc/manual/src/package-management/index.md
similarity index 100%
rename from doc/manual/src/package-management/package-management.md
rename to doc/manual/src/package-management/index.md
diff --git a/doc/manual/src/protocols/protocols.md b/doc/manual/src/protocols/index.md
similarity index 100%
rename from doc/manual/src/protocols/protocols.md
rename to doc/manual/src/protocols/index.md
diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md
index 1d2688ede..5f54abbde 100644
--- a/doc/manual/src/quick-start.md
+++ b/doc/manual/src/quick-start.md
@@ -13,7 +13,7 @@ to subsequent chapters.
    The install script will use `sudo`, so make sure you have sufficient rights.
    On Linux, `--daemon` can be omitted for a single-user install.
 
-   For other installation methods, see [here](installation/installation.md).
+   For other installation methods, see [here](installation/index.md).
 
 1. See what installable packages are currently available in the
    channel:
diff --git a/doc/manual/src/release-notes/release-notes.md b/doc/manual/src/release-notes/index.md
similarity index 100%
rename from doc/manual/src/release-notes/release-notes.md
rename to doc/manual/src/release-notes/index.md

From ea95327e72f5781295417b0eae46a5e351bebebd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 30 Nov 2023 16:16:17 +0100
Subject: [PATCH 107/421] Move restricted/pure-eval access control out of the
 evaluator and into the accessor

---
 src/libcmd/installables.cc             |   7 +-
 src/libexpr/eval.cc                    | 103 +++++----------------
 src/libexpr/eval.hh                    |  25 +++---
 src/libexpr/parser.y                   |  19 +++-
 src/libexpr/primops.cc                 | 119 +++++++++++--------------
 src/nix-build/nix-build.cc             |   7 +-
 src/nix-instantiate/nix-instantiate.cc |   2 +-
 tests/functional/restricted.sh         |   4 +-
 8 files changed, 115 insertions(+), 171 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 6e670efea..6b3c82374 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -260,9 +260,10 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s
 
             evalSettings.pureEval = false;
             auto state = getEvalState();
-            Expr *e = state->parseExprFromFile(
-                resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file)))
-                );
+            auto e =
+                state->parseExprFromFile(
+                    resolveExprPath(
+                        lookupFileArg(*state, *file)));
 
             Value root;
             state->eval(e, root);
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 7e68e6f9b..23ac349fe 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -509,7 +509,18 @@ EvalState::EvalState(
     , sOutputSpecified(symbols.create("outputSpecified"))
     , repair(NoRepair)
     , emptyBindings(0)
-    , rootFS(makeFSInputAccessor(CanonPath::root))
+    , rootFS(
+        makeFSInputAccessor(
+            CanonPath::root,
+            evalSettings.restrictEval || evalSettings.pureEval
+            ? std::optional<std::set<CanonPath>>(std::set<CanonPath>())
+            : std::nullopt,
+            [](const CanonPath & path) -> RestrictedPathError {
+                auto modeInformation = evalSettings.pureEval
+                    ? "in pure evaluation mode (use '--impure' to override)"
+                    : "in restricted mode";
+                throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation);
+            }))
     , corepkgsFS(makeMemoryInputAccessor())
     , internalFS(makeMemoryInputAccessor())
     , derivationInternal{corepkgsFS->addFile(
@@ -551,28 +562,10 @@ EvalState::EvalState(
             searchPath.elements.emplace_back(SearchPath::Elem::parse(i));
     }
 
-    if (evalSettings.restrictEval || evalSettings.pureEval) {
-        allowedPaths = PathSet();
-
-        for (auto & i : searchPath.elements) {
-            auto r = resolveSearchPathPath(i.path);
-            if (!r) continue;
-
-            auto path = std::move(*r);
-
-            if (store->isInStore(path)) {
-                try {
-                    StorePathSet closure;
-                    store->computeFSClosure(store->toStorePath(path).first, closure);
-                    for (auto & path : closure)
-                        allowPath(path);
-                } catch (InvalidPath &) {
-                    allowPath(path);
-                }
-            } else
-                allowPath(path);
-        }
-    }
+    /* Allow access to all paths in the search path. */
+    if (rootFS->hasAccessControl())
+        for (auto & i : searchPath.elements)
+            resolveSearchPathPath(i.path, true);
 
     corepkgsFS->addFile(
         CanonPath("fetchurl.nix"),
@@ -590,14 +583,12 @@ EvalState::~EvalState()
 
 void EvalState::allowPath(const Path & path)
 {
-    if (allowedPaths)
-        allowedPaths->insert(path);
+    rootFS->allowPath(CanonPath(path));
 }
 
 void EvalState::allowPath(const StorePath & storePath)
 {
-    if (allowedPaths)
-        allowedPaths->insert(store->toRealPath(storePath));
+    rootFS->allowPath(CanonPath(store->toRealPath(storePath)));
 }
 
 void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v)
@@ -607,54 +598,6 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value &
     mkStorePathString(storePath, v);
 }
 
-SourcePath EvalState::checkSourcePath(const SourcePath & path_)
-{
-    // Don't check non-rootFS accessors, they're in a different namespace.
-    if (path_.accessor != ref<InputAccessor>(rootFS)) return path_;
-
-    if (!allowedPaths) return path_;
-
-    auto i = resolvedPaths.find(path_.path.abs());
-    if (i != resolvedPaths.end())
-        return i->second;
-
-    bool found = false;
-
-    /* First canonicalize the path without symlinks, so we make sure an
-     * attacker can't append ../../... to a path that would be in allowedPaths
-     * and thus leak symlink targets.
-     */
-    Path abspath = canonPath(path_.path.abs());
-
-    for (auto & i : *allowedPaths) {
-        if (isDirOrInDir(abspath, i)) {
-            found = true;
-            break;
-        }
-    }
-
-    if (!found) {
-        auto modeInformation = evalSettings.pureEval
-            ? "in pure eval mode (use '--impure' to override)"
-            : "in restricted mode";
-        throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", abspath, modeInformation);
-    }
-
-    /* Resolve symlinks. */
-    debug("checking access to '%s'", abspath);
-    SourcePath path = rootPath(CanonPath(canonPath(abspath, true)));
-
-    for (auto & i : *allowedPaths) {
-        if (isDirOrInDir(path.path.abs(), i)) {
-            resolvedPaths.insert_or_assign(path_.path.abs(), path);
-            return path;
-        }
-    }
-
-    throw RestrictedPathError("access to canonical path '%1%' is forbidden in restricted mode", path);
-}
-
-
 void EvalState::checkURI(const std::string & uri)
 {
     if (!evalSettings.restrictEval) return;
@@ -674,12 +617,12 @@ void EvalState::checkURI(const std::string & uri)
     /* If the URI is a path, then check it against allowedPaths as
        well. */
     if (hasPrefix(uri, "/")) {
-        checkSourcePath(rootPath(CanonPath(uri)));
+        rootFS->checkAllowed(CanonPath(uri));
         return;
     }
 
     if (hasPrefix(uri, "file://")) {
-        checkSourcePath(rootPath(CanonPath(std::string(uri, 7))));
+        rootFS->checkAllowed(CanonPath(uri.substr(7)));
         return;
     }
 
@@ -1181,10 +1124,8 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env)
 }
 
 
-void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial)
+void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial)
 {
-    auto path = checkSourcePath(path_);
-
     FileEvalCache::iterator i;
     if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) {
         v = i->second;
@@ -1205,7 +1146,7 @@ void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial
         e = j->second;
 
     if (!e)
-        e = parseExprFromFile(checkSourcePath(resolvedPath));
+        e = parseExprFromFile(resolvedPath);
 
     fileParseCache[resolvedPath] = e;
 
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 9a92992c1..ee7bdda0d 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -217,12 +217,6 @@ public:
      */
     RepairFlag repair;
 
-    /**
-     * The allowed filesystem paths in restricted or pure evaluation
-     * mode.
-     */
-    std::optional<PathSet> allowedPaths;
-
     Bindings emptyBindings;
 
     /**
@@ -396,12 +390,6 @@ public:
      */
     void allowAndSetStorePathString(const StorePath & storePath, Value & v);
 
-    /**
-     * Check whether access to a path is allowed and throw an error if
-     * not. Otherwise return the canonicalised path.
-     */
-    SourcePath checkSourcePath(const SourcePath & path);
-
     void checkURI(const std::string & uri);
 
     /**
@@ -445,13 +433,15 @@ public:
     SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
 
     /**
-     * Try to resolve a search path value (not the optional key part)
+     * Try to resolve a search path value (not the optional key part).
      *
      * If the specified search path element is a URI, download it.
      *
      * If it is not found, return `std::nullopt`
      */
-    std::optional<std::string> resolveSearchPathPath(const SearchPath::Path & path);
+    std::optional<std::string> resolveSearchPathPath(
+        const SearchPath::Path & elem,
+        bool initAccessControl = false);
 
     /**
      * Evaluate an expression to normal form
@@ -756,6 +746,13 @@ public:
      */
     [[nodiscard]] StringMap realiseContext(const NixStringContext & context);
 
+    /* Call the binary path filter predicate used builtins.path etc. */
+    bool callPathFilter(
+        Value * filterFun,
+        const SourcePath & path,
+        std::string_view pathArg,
+        PosIdx pos);
+
 private:
 
     /**
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index f6cf1f689..58fc580fc 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -783,7 +783,7 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_
 }
 
 
-std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Path & value0)
+std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Path & value0, bool initAccessControl)
 {
     auto & value = value0.s;
     auto i = searchPathResolved.find(value);
@@ -800,7 +800,6 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
             logWarning({
                 .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value)
             });
-            res = std::nullopt;
         }
     }
 
@@ -814,6 +813,20 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
 
     else {
         auto path = absPath(value);
+
+        /* Allow access to paths in the search path. */
+        if (initAccessControl) {
+            allowPath(path);
+            if (store->isInStore(path)) {
+                try {
+                    StorePathSet closure;
+                    store->computeFSClosure(store->toStorePath(path).first, closure);
+                    for (auto & p : closure)
+                        allowPath(p);
+                } catch (InvalidPath &) { }
+            }
+        }
+
         if (pathExists(path))
             res = { path };
         else {
@@ -829,7 +842,7 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
     else
         debug("failed to resolve search path element '%s'", value);
 
-    searchPathResolved[value] = res;
+    searchPathResolved.emplace(value, res);
     return res;
 }
 
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index ebf2549e4..0f7706563 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -15,6 +15,7 @@
 #include "value-to-json.hh"
 #include "value-to-xml.hh"
 #include "primops.hh"
+#include "fs-input-accessor.hh"
 
 #include <boost/container/small_vector.hpp>
 #include <boost/regex.hpp>
@@ -90,8 +91,8 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
         for (auto & [outputName, outputPath] : outputs) {
             /* Add the output of this derivations to the allowed
                paths. */
-            if (allowedPaths) {
-                allowPath(outputPath);
+            if (rootFS->hasAccessControl()) {
+                allowPath(store->toRealPath(outputPath));
             }
             /* Get all the output paths corresponding to the placeholders we had */
             if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
@@ -110,27 +111,19 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
     return res;
 }
 
-struct RealisePathFlags {
-    // Whether to check that the path is allowed in pure eval mode
-    bool checkForPureEval = true;
-};
-
-static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, const RealisePathFlags flags = {})
+static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v)
 {
     NixStringContext context;
 
     auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path");
 
     try {
-        if (!context.empty()) {
+        if (!context.empty() && path.accessor == state.rootFS) {
             auto rewrites = state.realiseContext(context);
             auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context);
             return {path.accessor, CanonPath(realPath)};
-        }
-
-        return flags.checkForPureEval
-            ? state.checkSourcePath(path)
-            : path;
+        } else
+            return path;
     } catch (Error & e) {
         e.addTrace(state.positions[pos], "while realising the context of path '%s'", path);
         throw;
@@ -1493,7 +1486,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args,
         }));
 
     NixStringContext context;
-    auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'")).path;
+    auto path = state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'").path;
     /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink
        directly in the store.  The latter condition is necessary so
        e.g. nix-push does the right thing. */
@@ -1535,12 +1528,7 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args,
 {
     auto & arg = *args[0];
 
-    /* We don’t check the path right now, because we don’t want to
-       throw if the path isn’t allowed, but just return false (and we
-       can’t just catch the exception here because we still want to
-       throw if something in the evaluation of `arg` tries to
-       access an unauthorized path). */
-    auto path = realisePath(state, pos, arg, { .checkForPureEval = false });
+    auto path = realisePath(state, pos, arg);
 
     /* SourcePath doesn't know about trailing slash. */
     auto mustBeDir = arg.type() == nString
@@ -1548,14 +1536,9 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args,
             || arg.string_view().ends_with("/."));
 
     try {
-        auto checked = state.checkSourcePath(path);
-        auto st = checked.maybeLstat();
+        auto st = path.maybeLstat();
         auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory);
         v.mkBool(exists);
-    } catch (SysError & e) {
-        /* Don't give away info from errors while canonicalising
-           ‘path’ in restricted mode. */
-        v.mkBool(false);
     } catch (RestrictedPathError & e) {
         v.mkBool(false);
     }
@@ -1699,7 +1682,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V
 
     auto path = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.findFile");
 
-    v.mkPath(state.checkSourcePath(state.findFile(searchPath, path, pos)));
+    v.mkPath(state.findFile(searchPath, path, pos));
 }
 
 static RegisterPrimOp primop_findFile(PrimOp {
@@ -2178,11 +2161,35 @@ static RegisterPrimOp primop_toFile({
     .fun = prim_toFile,
 });
 
+bool EvalState::callPathFilter(
+    Value * filterFun,
+    const SourcePath & path,
+    std::string_view pathArg,
+    PosIdx pos)
+{
+    auto st = path.lstat();
+
+    /* Call the filter function.  The first argument is the path, the
+       second is a string indicating the type of the file. */
+    Value arg1;
+    arg1.mkString(pathArg);
+
+    Value arg2;
+    // assert that type is not "unknown"
+    arg2.mkString(fileTypeToString(st.type));
+
+    Value * args []{&arg1, &arg2};
+    Value res;
+    callFunction(*filterFun, 2, args, res, pos);
+
+    return forceBool(res, pos, "while evaluating the return value of the path filter function");
+}
+
 static void addPath(
     EvalState & state,
     const PosIdx pos,
     std::string_view name,
-    Path path,
+    SourcePath path,
     Value * filterFun,
     FileIngestionMethod method,
     const std::optional<Hash> expectedHash,
@@ -2190,48 +2197,29 @@ static void addPath(
     const NixStringContext & context)
 {
     try {
-        // FIXME: handle CA derivation outputs (where path needs to
-        // be rewritten to the actual output).
-        auto rewrites = state.realiseContext(context);
-        path = state.toRealPath(rewriteStrings(path, rewrites), context);
-
         StorePathSet refs;
 
-        if (state.store->isInStore(path)) {
+        if (path.accessor == state.rootFS && state.store->isInStore(path.path.abs())) {
+            // FIXME: handle CA derivation outputs (where path needs to
+            // be rewritten to the actual output).
+            auto rewrites = state.realiseContext(context);
+            path = {state.rootFS, CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context))};
+
             try {
-                auto [storePath, subPath] = state.store->toStorePath(path);
+                auto [storePath, subPath] = state.store->toStorePath(path.path.abs());
                 // FIXME: we should scanForReferences on the path before adding it
                 refs = state.store->queryPathInfo(storePath)->references;
-                path = state.store->toRealPath(storePath) + subPath;
+                path = {state.rootFS, CanonPath(state.store->toRealPath(storePath) + subPath)};
             } catch (Error &) { // FIXME: should be InvalidPathError
             }
         }
 
-        path = evalSettings.pureEval && expectedHash
-            ? path
-            : state.checkSourcePath(state.rootPath(CanonPath(path))).path.abs();
-
-        PathFilter filter = filterFun ? ([&](const Path & path) {
-            auto st = lstat(path);
-
-            /* Call the filter function.  The first argument is the path,
-               the second is a string indicating the type of the file. */
-            Value arg1;
-            arg1.mkString(path);
-
-            Value arg2;
-            arg2.mkString(
-                S_ISREG(st.st_mode) ? "regular" :
-                S_ISDIR(st.st_mode) ? "directory" :
-                S_ISLNK(st.st_mode) ? "symlink" :
-                "unknown" /* not supported, will fail! */);
-
-            Value * args []{&arg1, &arg2};
-            Value res;
-            state.callFunction(*filterFun, 2, args, res, pos);
-
-            return state.forceBool(res, pos, "while evaluating the return value of the path filter function");
-        }) : defaultPathFilter;
+        std::unique_ptr<PathFilter> filter;
+        if (filterFun)
+            filter = std::make_unique<PathFilter>([&](const Path & p) {
+                auto p2 = CanonPath(p);
+                return state.callPathFilter(filterFun, {path.accessor, p2}, p2.abs(), pos);
+            });
 
         std::optional<StorePath> expectedStorePath;
         if (expectedHash)
@@ -2242,7 +2230,7 @@ static void addPath(
             });
 
         if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
-            auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, method, &filter, state.repair);
+            auto dstPath = path.fetchToStore(state.store, name, method, filter.get(), state.repair);
             if (expectedHash && expectedStorePath != dstPath)
                 state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path));
             state.allowAndSetStorePathString(dstPath, v);
@@ -2261,7 +2249,8 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg
     auto path = state.coerceToPath(pos, *args[1], context,
         "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'");
     state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource");
-    addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
+
+    addPath(state, pos, path.baseName(), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
 }
 
 static RegisterPrimOp primop_filterSource({
@@ -2356,7 +2345,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value
     if (name.empty())
         name = path->baseName();
 
-    addPath(state, pos, name, path->path.abs(), filterFun, method, expectedHash, v, context);
+    addPath(state, pos, name, *path, filterFun, method, expectedHash, v, context);
 }
 
 static RegisterPrimOp primop_path({
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 75ce12a8c..e2986bfe0 100644
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -310,8 +310,11 @@ static void main_nix_build(int argc, char * * argv)
                 else
                     /* If we're in a #! script, interpret filenames
                        relative to the script. */
-                    exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state,
-                                        inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i)))));
+                    exprs.push_back(
+                        state->parseExprFromFile(
+                            resolveExprPath(
+                                lookupFileArg(*state,
+                                    inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))));
             }
         }
 
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index c67409e89..86b9be17d 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -183,7 +183,7 @@ static int main_nix_instantiate(int argc, char * * argv)
         for (auto & i : files) {
             Expr * e = fromArgs
                 ? state->parseExprFromString(i, state->rootPath(CanonPath::fromCwd()))
-                : state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, i))));
+                : state->parseExprFromFile(resolveExprPath(lookupFileArg(*state, i)));
             processExpr(*state, attrPaths, parseOnly, strict, autoArgs,
                 evalOnly, outputKind, xmlOutputSourceLocation, e);
         }
diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh
index 197ae7a10..b8deceacc 100644
--- a/tests/functional/restricted.sh
+++ b/tests/functional/restricted.sh
@@ -14,8 +14,8 @@ nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I sr
 (! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel')
 nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel' -I src=../../src
 
-(! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>')
-nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>' -I src=.
+(! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile <foo/simple.nix>')
+nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile <foo/simple.nix>' -I src=.
 
 p=$(nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval --allowed-uris "file://$(pwd)")
 cmp $p restricted.sh

From 305939655a6cd680997981ca6077d4ce7f957984 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 30 Nov 2023 16:28:33 +0100
Subject: [PATCH 108/421] Remove superfluous use of hasAccessControl()

---
 src/libexpr/primops.cc | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 0f7706563..c442de986 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -91,9 +91,8 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
         for (auto & [outputName, outputPath] : outputs) {
             /* Add the output of this derivations to the allowed
                paths. */
-            if (rootFS->hasAccessControl()) {
-                allowPath(store->toRealPath(outputPath));
-            }
+            allowPath(store->toRealPath(outputPath));
+
             /* Get all the output paths corresponding to the placeholders we had */
             if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
                 res.insert_or_assign(

From 43d9fb6cf180c421be17b4247f5dd032cf4843f5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 30 Nov 2023 16:44:54 +0100
Subject: [PATCH 109/421] Remove InputAccessor::root()

---
 src/libexpr/value.hh              | 7 +++----
 src/libfetchers/fetchers.cc       | 2 +-
 src/libfetchers/input-accessor.cc | 7 +------
 src/libfetchers/input-accessor.hh | 7 +++++--
 4 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index bcff8ae55..72a3a2b32 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -424,10 +424,9 @@ public:
     SourcePath path() const
     {
         assert(internalType == tPath);
-        return SourcePath {
-            .accessor = ref(_path.accessor->shared_from_this()),
-            .path = CanonPath(CanonPath::unchecked_t(), _path.path)
-        };
+        return SourcePath(
+            ref(_path.accessor->shared_from_this()),
+            CanonPath(CanonPath::unchecked_t(), _path.path));
     }
 
     std::string_view string_view() const
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 60208619e..5fd9e069f 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -374,7 +374,7 @@ void InputScheme::clone(const Input & input, const Path & destDir) const
 std::pair<StorePath, Input> InputScheme::fetch(ref<Store> store, const Input & input)
 {
     auto [accessor, input2] = getAccessor(store, input);
-    auto storePath = accessor->root().fetchToStore(store, input2.getName());
+    auto storePath = SourcePath(accessor).fetchToStore(store, input2.getName());
     return {storePath, input2};
 }
 
diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc
index 85dc4609f..f54a5a6fd 100644
--- a/src/libfetchers/input-accessor.cc
+++ b/src/libfetchers/input-accessor.cc
@@ -53,11 +53,6 @@ StorePath InputAccessor::fetchToStore(
     return storePath;
 }
 
-SourcePath InputAccessor::root()
-{
-    return {ref(shared_from_this()), CanonPath::root};
-}
-
 std::ostream & operator << (std::ostream & str, const SourcePath & path)
 {
     str << path.to_string();
@@ -88,7 +83,7 @@ SourcePath SourcePath::parent() const
 
 SourcePath SourcePath::resolveSymlinks() const
 {
-    auto res = accessor->root();
+    auto res = SourcePath(accessor);
 
     int linksAllowed = 1024;
 
diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh
index 26d17f064..d5ac238b1 100644
--- a/src/libfetchers/input-accessor.hh
+++ b/src/libfetchers/input-accessor.hh
@@ -36,8 +36,6 @@ struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this<Inpu
         FileIngestionMethod method = FileIngestionMethod::Recursive,
         PathFilter * filter = nullptr,
         RepairFlag repair = NoRepair);
-
-    SourcePath root();
 };
 
 /**
@@ -51,6 +49,11 @@ struct SourcePath
     ref<InputAccessor> accessor;
     CanonPath path;
 
+    SourcePath(ref<InputAccessor> accessor, CanonPath path = CanonPath::root)
+        : accessor(std::move(accessor))
+        , path(std::move(path))
+    { }
+
     std::string_view baseName() const;
 
     /**

From be30c2ea8de7f12d610b695bd7c6edf22b32fe55 Mon Sep 17 00:00:00 2001
From: Rebecca Turner <rbt@sent.as>
Date: Wed, 8 Nov 2023 17:52:22 -0800
Subject: [PATCH 110/421] Don't attempt to `git add` ignored files

This uses `git check-ignore` to determine if files are ignored before
attempting to add them in `putFile`.

We also add a condition to the `fetchFromWorkdir` filter to always add
the `flake.lock` file, even if it's not tracked. This is necessary to
resolve inputs.

This fixes #8854 without `git add --force`.
---
 src/libfetchers/git.cc | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 8cd74057c..734c29258 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -313,15 +313,26 @@ struct GitInputScheme : InputScheme
 
         writeFile((CanonPath(repoInfo.url) + path).abs(), contents);
 
-        runProgram("git", true,
-            { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) });
+        auto result = runProgram(RunOptions {
+            .program = "git",
+            .args = {"-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "check-ignore", "--quiet", std::string(path.rel())},
+        });
+        auto exitCode = WEXITSTATUS(result.first);
 
-        // Pause the logger to allow for user input (such as a gpg passphrase) in `git commit`
-        logger->pause();
-        Finally restoreLogger([]() { logger->resume(); });
-        if (commitMsg)
+        if (exitCode != 0) {
+            // The path is not `.gitignore`d, we can add the file.
             runProgram("git", true,
-                { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-m", *commitMsg });
+                { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) });
+
+
+            if (commitMsg) {
+                // Pause the logger to allow for user input (such as a gpg passphrase) in `git commit`
+                logger->pause();
+                Finally restoreLogger([]() { logger->resume(); });
+                runProgram("git", true,
+                    { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-m", *commitMsg });
+            }
+        }
     }
 
     struct RepoInfo

From 44d21f6ef9783bed8812d39ff7b1a28a4883f84b Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Thu, 30 Nov 2023 20:36:50 +0100
Subject: [PATCH 111/421] keep generated documentation in a separate directory

- helps navigating the code as it highlights which files are generated
- makes it less error prone when working incrementally
  (although this should be just fixed by building out of tree)
---
 .gitignore                                    |  2 +-
 .../manual/src/store/types/index.md.in        |  4 ----
 src/nix/local.mk                              | 19 ++++++++++++++++---
 src/nix/main.cc                               |  2 +-
 src/nix/profile.md                            |  2 +-
 5 files changed, 19 insertions(+), 10 deletions(-)
 rename src/nix/help-stores.md => doc/manual/src/store/types/index.md.in (99%)

diff --git a/.gitignore b/.gitignore
index 13a4dfb75..bcf9c4a01 100644
--- a/.gitignore
+++ b/.gitignore
@@ -54,7 +54,7 @@ perl/Makefile.config
 
 /src/nix/nix
 
-/src/nix/doc
+/src/nix/generated-doc
 
 # /src/nix-env/
 /src/nix-env/nix-env
diff --git a/src/nix/help-stores.md b/doc/manual/src/store/types/index.md.in
similarity index 99%
rename from src/nix/help-stores.md
rename to doc/manual/src/store/types/index.md.in
index 47ba9b94d..bb166a1fc 100644
--- a/src/nix/help-stores.md
+++ b/doc/manual/src/store/types/index.md.in
@@ -1,5 +1,3 @@
-R"(
-
 Nix supports different types of stores. These are described below.
 
 ## Store URL format
@@ -42,5 +40,3 @@ store as follows:
 * Otherwise, use the [local store](#local-store) `/nix/store`.
 
 @stores@
-
-)"
diff --git a/src/nix/local.mk b/src/nix/local.mk
index 57f8259c4..a21aa705f 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -31,10 +31,23 @@ src/nix/develop.cc: src/nix/get-env.sh.gen.hh
 
 src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh
 
-src/nix/main.cc: doc/manual/generate-manpage.nix.gen.hh doc/manual/utils.nix.gen.hh doc/manual/generate-settings.nix.gen.hh doc/manual/generate-store-info.nix.gen.hh
+src/nix/main.cc: \
+  doc/manual/generate-manpage.nix.gen.hh \
+  doc/manual/utils.nix.gen.hh doc/manual/generate-settings.nix.gen.hh \
+  doc/manual/generate-store-info.nix.gen.hh \
+  src/nix/generated-doc/help-stores.md
 
-src/nix/doc/files/%.md: doc/manual/src/command-ref/files/%.md
+src/nix/generated-doc/files/%.md: doc/manual/src/command-ref/files/%.md
 	@mkdir -p $$(dirname $@)
 	@cp $< $@
 
-src/nix/profile.cc: src/nix/profile.md src/nix/doc/files/profiles.md.gen.hh
+src/nix/profile.cc: src/nix/profile.md src/nix/generated-doc/files/profiles.md.gen.hh
+
+src/nix/generated-doc/help-stores.md: doc/manual/src/store/types/index.md.in
+	@mkdir -p $$(dirname $@)
+	@echo 'R"(' >> $@.tmp
+	@echo >> $@.tmp
+	@cat $^ >> $@.tmp
+	@echo >> $@.tmp
+	@echo ')"' >> $@.tmp
+	@mv $@.tmp $@
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 2a6c2f478..49e637fb0 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -297,7 +297,7 @@ struct CmdHelpStores : Command
     std::string doc() override
     {
         return
-          #include "help-stores.md"
+          #include "generated-doc/help-stores.md"
           ;
     }
 
diff --git a/src/nix/profile.md b/src/nix/profile.md
index bd13f906f..9b2f86f4a 100644
--- a/src/nix/profile.md
+++ b/src/nix/profile.md
@@ -11,7 +11,7 @@ them to be rolled back easily.
 
 )""
 
-#include "doc/files/profiles.md.gen.hh"
+#include "generated-doc/files/profiles.md.gen.hh"
 
 R""(
 

From d5ffc94f336fc4032dd4009c14c148e390e10e16 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Thu, 30 Nov 2023 21:41:47 +0100
Subject: [PATCH 112/421] use lookup paths in helper expressions consistently

this makes the files in question a bit more independent of source location.

to find where the value is set and how it's wired up:

    rg nix=doc/manual
---
 doc/manual/generate-builtin-constants.nix     |  2 +-
 doc/manual/generate-builtins.nix              |  2 +-
 doc/manual/generate-manpage.nix               | 30 +++++++++++++++----
 doc/manual/generate-settings.nix              |  2 +-
 doc/manual/generate-store-info.nix            |  4 +--
 doc/manual/generate-xp-features-shortlist.nix |  2 +-
 doc/manual/generate-xp-features.nix           |  2 +-
 7 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/doc/manual/generate-builtin-constants.nix b/doc/manual/generate-builtin-constants.nix
index 8af80a02c..cccd1e279 100644
--- a/doc/manual/generate-builtin-constants.nix
+++ b/doc/manual/generate-builtin-constants.nix
@@ -1,6 +1,6 @@
 let
   inherit (builtins) concatStringsSep attrValues mapAttrs;
-  inherit (import ./utils.nix) optionalString squash;
+  inherit (import <nix/utils.nix>) optionalString squash;
 in
 
 builtinsInfo:
diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix
index 813a287f5..05cae1c46 100644
--- a/doc/manual/generate-builtins.nix
+++ b/doc/manual/generate-builtins.nix
@@ -1,6 +1,6 @@
 let
   inherit (builtins) concatStringsSep attrValues mapAttrs;
-  inherit (import ./utils.nix) optionalString squash;
+  inherit (import <nix/utils.nix>) optionalString squash;
 in
 
 builtinsInfo:
diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix
index 14136016d..c4b9d1335 100644
--- a/doc/manual/generate-manpage.nix
+++ b/doc/manual/generate-manpage.nix
@@ -1,9 +1,29 @@
 let
   inherit (builtins)
-    attrNames attrValues fromJSON listToAttrs mapAttrs groupBy
-    concatStringsSep concatMap length lessThan replaceStrings sort;
-  inherit (import <nix/utils.nix>) attrsToList concatStrings optionalString filterAttrs trim squash unique;
-  showStoreDocs = import ./generate-store-info.nix;
+    attrNames
+    attrValues
+    concatMap
+    concatStringsSep
+    fromJSON
+    groupBy
+    length
+    lessThan
+    listToAttrs
+    mapAttrs
+    match
+    replaceStrings
+    sort
+    ;
+  inherit (import <nix/utils.nix>)
+    attrsToList
+    concatStrings
+    filterAttrs
+    optionalString
+    squash
+    trim
+    unique
+    ;
+  showStoreDocs = import <nix/generate-store-info.nix>;
 in
 
 inlineHTML: commandDump:
@@ -97,7 +117,7 @@ let
             ${optionalString (cat != "") "## ${cat}"}
 
             ${concatStringsSep "\n" (attrValues (mapAttrs showOption opts))}
-            '';
+          '';
           showOption = name: option:
             let
               result = trim ''
diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix
index 8736bb793..3add10075 100644
--- a/doc/manual/generate-settings.nix
+++ b/doc/manual/generate-settings.nix
@@ -1,6 +1,6 @@
 let
   inherit (builtins) attrValues concatStringsSep isAttrs isBool mapAttrs;
-  inherit (import ./utils.nix) concatStrings indent optionalString squash;
+  inherit (import <nix/utils.nix>) concatStrings indent optionalString squash;
 in
 
 # `inlineHTML` is a hack to accommodate inconsistent output from `lowdown`
diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix
index 36215aadf..73defcb71 100644
--- a/doc/manual/generate-store-info.nix
+++ b/doc/manual/generate-store-info.nix
@@ -1,7 +1,7 @@
 let
   inherit (builtins) attrValues mapAttrs;
-  inherit (import ./utils.nix) concatStrings optionalString;
-  showSettings = import ./generate-settings.nix;
+  inherit (import <nix/utils.nix>) concatStrings optionalString;
+  showSettings = import <nix/generate-settings.nix>;
 in
 
 inlineHTML: storesInfo:
diff --git a/doc/manual/generate-xp-features-shortlist.nix b/doc/manual/generate-xp-features-shortlist.nix
index 30e211c96..ec09f4b75 100644
--- a/doc/manual/generate-xp-features-shortlist.nix
+++ b/doc/manual/generate-xp-features-shortlist.nix
@@ -1,5 +1,5 @@
 with builtins;
-with import ./utils.nix;
+with import <nix/utils.nix>;
 
 let
   showExperimentalFeature = name: doc:
diff --git a/doc/manual/generate-xp-features.nix b/doc/manual/generate-xp-features.nix
index adb94355c..fc7c7d4cf 100644
--- a/doc/manual/generate-xp-features.nix
+++ b/doc/manual/generate-xp-features.nix
@@ -1,5 +1,5 @@
 with builtins;
-with import ./utils.nix;
+with import <nix/utils.nix>;
 
 let
   showExperimentalFeature = name: doc:

From 8cafc754d845529a78595d1196769257ee23ca56 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 30 Nov 2023 21:54:53 +0100
Subject: [PATCH 113/421] Move access control from FSInputAccessor to
 FilteringInputAccessor

---
 src/libexpr/eval.cc                         | 23 +++---
 src/libexpr/eval.hh                         |  3 +-
 src/libfetchers/filtering-input-accessor.cc | 83 +++++++++++++++++++++
 src/libfetchers/filtering-input-accessor.hh | 73 ++++++++++++++++++
 src/libfetchers/fs-input-accessor.cc        | 77 +++----------------
 src/libfetchers/fs-input-accessor.hh        | 22 +-----
 src/libfetchers/git.cc                      |  6 +-
 7 files changed, 191 insertions(+), 96 deletions(-)
 create mode 100644 src/libfetchers/filtering-input-accessor.cc
 create mode 100644 src/libfetchers/filtering-input-accessor.hh

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 23ac349fe..841c223cd 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -14,6 +14,7 @@
 #include "profiles.hh"
 #include "print.hh"
 #include "fs-input-accessor.hh"
+#include "filtering-input-accessor.hh"
 #include "memory-input-accessor.hh"
 #include "signals.hh"
 #include "gc-small-vector.hh"
@@ -510,17 +511,15 @@ EvalState::EvalState(
     , repair(NoRepair)
     , emptyBindings(0)
     , rootFS(
-        makeFSInputAccessor(
-            CanonPath::root,
-            evalSettings.restrictEval || evalSettings.pureEval
-            ? std::optional<std::set<CanonPath>>(std::set<CanonPath>())
-            : std::nullopt,
+        evalSettings.restrictEval || evalSettings.pureEval
+        ? ref<InputAccessor>(AllowListInputAccessor::create(makeFSInputAccessor(CanonPath::root), {},
             [](const CanonPath & path) -> RestrictedPathError {
                 auto modeInformation = evalSettings.pureEval
                     ? "in pure evaluation mode (use '--impure' to override)"
                     : "in restricted mode";
                 throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation);
             }))
+        : makeFSInputAccessor(CanonPath::root))
     , corepkgsFS(makeMemoryInputAccessor())
     , internalFS(makeMemoryInputAccessor())
     , derivationInternal{corepkgsFS->addFile(
@@ -563,7 +562,7 @@ EvalState::EvalState(
     }
 
     /* Allow access to all paths in the search path. */
-    if (rootFS->hasAccessControl())
+    if (rootFS.dynamic_pointer_cast<AllowListInputAccessor>())
         for (auto & i : searchPath.elements)
             resolveSearchPathPath(i.path, true);
 
@@ -583,12 +582,14 @@ EvalState::~EvalState()
 
 void EvalState::allowPath(const Path & path)
 {
-    rootFS->allowPath(CanonPath(path));
+    if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListInputAccessor>())
+        rootFS2->allowPath(CanonPath(path));
 }
 
 void EvalState::allowPath(const StorePath & storePath)
 {
-    rootFS->allowPath(CanonPath(store->toRealPath(storePath)));
+    if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListInputAccessor>())
+        rootFS2->allowPath(CanonPath(store->toRealPath(storePath)));
 }
 
 void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v)
@@ -617,12 +618,14 @@ void EvalState::checkURI(const std::string & uri)
     /* If the URI is a path, then check it against allowedPaths as
        well. */
     if (hasPrefix(uri, "/")) {
-        rootFS->checkAllowed(CanonPath(uri));
+        if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListInputAccessor>())
+            rootFS2->checkAccess(CanonPath(uri));
         return;
     }
 
     if (hasPrefix(uri, "file://")) {
-        rootFS->checkAllowed(CanonPath(uri.substr(7)));
+        if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListInputAccessor>())
+            rootFS2->checkAccess(CanonPath(uri.substr(7)));
         return;
     }
 
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index ee7bdda0d..f3f6d35b9 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -30,7 +30,6 @@ class EvalState;
 class StorePath;
 struct SingleDerivedPath;
 enum RepairFlag : bool;
-struct FSInputAccessor;
 struct MemoryInputAccessor;
 
 
@@ -222,7 +221,7 @@ public:
     /**
      * The accessor for the root filesystem.
      */
-    const ref<FSInputAccessor> rootFS;
+    const ref<InputAccessor> rootFS;
 
     /**
      * The in-memory filesystem for <nix/...> paths.
diff --git a/src/libfetchers/filtering-input-accessor.cc b/src/libfetchers/filtering-input-accessor.cc
new file mode 100644
index 000000000..5ae416fd3
--- /dev/null
+++ b/src/libfetchers/filtering-input-accessor.cc
@@ -0,0 +1,83 @@
+#include "filtering-input-accessor.hh"
+
+namespace nix {
+
+std::string FilteringInputAccessor::readFile(const CanonPath & path)
+{
+    checkAccess(path);
+    return next->readFile(prefix + path);
+}
+
+bool FilteringInputAccessor::pathExists(const CanonPath & path)
+{
+    return isAllowed(path) && next->pathExists(prefix + path);
+}
+
+std::optional<InputAccessor::Stat> FilteringInputAccessor::maybeLstat(const CanonPath & path)
+{
+    checkAccess(path);
+    return next->maybeLstat(prefix + path);
+}
+
+InputAccessor::DirEntries FilteringInputAccessor::readDirectory(const CanonPath & path)
+{
+    checkAccess(path);
+    DirEntries entries;
+    for (auto & entry : next->readDirectory(prefix + path)) {
+        if (isAllowed(path + entry.first))
+            entries.insert(std::move(entry));
+    }
+    return entries;
+}
+
+std::string FilteringInputAccessor::readLink(const CanonPath & path)
+{
+    checkAccess(path);
+    return next->readLink(prefix + path);
+}
+
+std::string FilteringInputAccessor::showPath(const CanonPath & path)
+{
+    return next->showPath(prefix + path);
+}
+
+void FilteringInputAccessor::checkAccess(const CanonPath & path)
+{
+    if (!isAllowed(path))
+        throw makeNotAllowedError
+            ? makeNotAllowedError(path)
+            : RestrictedPathError("access to path '%s' is forbidden", showPath(path));
+}
+
+struct AllowListInputAccessorImpl : AllowListInputAccessor
+{
+    std::set<CanonPath> allowedPaths;
+
+    AllowListInputAccessorImpl(
+        ref<InputAccessor> next,
+        std::set<CanonPath> && allowedPaths,
+        MakeNotAllowedError && makeNotAllowedError)
+        : AllowListInputAccessor(SourcePath(next), std::move(makeNotAllowedError))
+        , allowedPaths(std::move(allowedPaths))
+    { }
+
+    bool isAllowed(const CanonPath & path) override
+    {
+        return path.isAllowed(allowedPaths);
+    }
+
+    void allowPath(CanonPath path) override
+    {
+        allowedPaths.insert(std::move(path));
+    }
+};
+
+ref<AllowListInputAccessor> AllowListInputAccessor::create(
+    ref<InputAccessor> next,
+    std::set<CanonPath> && allowedPaths,
+    MakeNotAllowedError && makeNotAllowedError)
+{
+    return make_ref<AllowListInputAccessorImpl>(next, std::move(allowedPaths), std::move(makeNotAllowedError));
+}
+
+}
diff --git a/src/libfetchers/filtering-input-accessor.hh b/src/libfetchers/filtering-input-accessor.hh
new file mode 100644
index 000000000..209d26974
--- /dev/null
+++ b/src/libfetchers/filtering-input-accessor.hh
@@ -0,0 +1,73 @@
+#pragma once
+
+#include "input-accessor.hh"
+
+namespace nix {
+
+/**
+ * A function that should throw an exception of type
+ * `RestrictedPathError` explaining that access to `path` is
+ * forbidden.
+ */
+typedef std::function<RestrictedPathError(const CanonPath & path)> MakeNotAllowedError;
+
+/**
+ * An abstract wrapping `InputAccessor` that performs access
+ * control. Subclasses should override `checkAccess()` to implement an
+ * access control policy.
+ */
+struct FilteringInputAccessor : InputAccessor
+{
+    ref<InputAccessor> next;
+    CanonPath prefix;
+    MakeNotAllowedError makeNotAllowedError;
+
+    FilteringInputAccessor(const SourcePath & src, MakeNotAllowedError && makeNotAllowedError)
+        : next(src.accessor)
+        , prefix(src.path)
+        , makeNotAllowedError(std::move(makeNotAllowedError))
+    { }
+
+    std::string readFile(const CanonPath & path) override;
+
+    bool pathExists(const CanonPath & path) override;
+
+    std::optional<Stat> maybeLstat(const CanonPath & path) override;
+
+    DirEntries readDirectory(const CanonPath & path) override;
+
+    std::string readLink(const CanonPath & path) override;
+
+    std::string showPath(const CanonPath & path) override;
+
+    /**
+     * Call `makeNotAllowedError` to throw a `RestrictedPathError`
+     * exception if `isAllowed()` returns `false` for `path`.
+     */
+    void checkAccess(const CanonPath & path);
+
+    /**
+     * Return `true` iff access to path is allowed.
+     */
+    virtual bool isAllowed(const CanonPath & path) = 0;
+};
+
+/**
+ * A wrapping `InputAccessor` that checks paths against an allow-list.
+ */
+struct AllowListInputAccessor : public FilteringInputAccessor
+{
+    /**
+     * Grant access to the specified path.
+     */
+    virtual void allowPath(CanonPath path) = 0;
+
+    static ref<AllowListInputAccessor> create(
+        ref<InputAccessor> next,
+        std::set<CanonPath> && allowedPaths,
+        MakeNotAllowedError && makeNotAllowedError);
+
+    using FilteringInputAccessor::FilteringInputAccessor;
+};
+
+}
diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc
index 2efee932d..c3d8d273c 100644
--- a/src/libfetchers/fs-input-accessor.cc
+++ b/src/libfetchers/fs-input-accessor.cc
@@ -4,19 +4,12 @@
 
 namespace nix {
 
-struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor
+struct FSInputAccessor : InputAccessor, PosixSourceAccessor
 {
     CanonPath root;
-    std::optional<std::set<CanonPath>> allowedPaths;
-    MakeNotAllowedError makeNotAllowedError;
 
-    FSInputAccessorImpl(
-        const CanonPath & root,
-        std::optional<std::set<CanonPath>> && allowedPaths,
-        MakeNotAllowedError && makeNotAllowedError)
+    FSInputAccessor(const CanonPath & root)
         : root(root)
-        , allowedPaths(std::move(allowedPaths))
-        , makeNotAllowedError(std::move(makeNotAllowedError))
     {
         displayPrefix = root.isRoot() ? "" : root.abs();
     }
@@ -27,39 +20,30 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor
         std::function<void(uint64_t)> sizeCallback) override
     {
         auto absPath = makeAbsPath(path);
-        checkAllowed(absPath);
         PosixSourceAccessor::readFile(absPath, sink, sizeCallback);
     }
 
     bool pathExists(const CanonPath & path) override
     {
-        auto absPath = makeAbsPath(path);
-        return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath);
+        return PosixSourceAccessor::pathExists(makeAbsPath(path));
     }
 
     std::optional<Stat> maybeLstat(const CanonPath & path) override
     {
-        auto absPath = makeAbsPath(path);
-        checkAllowed(absPath);
-        return PosixSourceAccessor::maybeLstat(absPath);
+        return PosixSourceAccessor::maybeLstat(makeAbsPath(path));
     }
 
     DirEntries readDirectory(const CanonPath & path) override
     {
-        auto absPath = makeAbsPath(path);
-        checkAllowed(absPath);
         DirEntries res;
-        for (auto & entry : PosixSourceAccessor::readDirectory(absPath))
-            if (isAllowed(absPath + entry.first))
-                res.emplace(entry);
+        for (auto & entry : PosixSourceAccessor::readDirectory(makeAbsPath(path)))
+            res.emplace(entry);
         return res;
     }
 
     std::string readLink(const CanonPath & path) override
     {
-        auto absPath = makeAbsPath(path);
-        checkAllowed(absPath);
-        return PosixSourceAccessor::readLink(absPath);
+        return PosixSourceAccessor::readLink(makeAbsPath(path));
     }
 
     CanonPath makeAbsPath(const CanonPath & path)
@@ -67,59 +51,22 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor
         return root + path;
     }
 
-    void checkAllowed(const CanonPath & absPath) override
-    {
-        if (!isAllowed(absPath))
-            throw makeNotAllowedError
-                ? makeNotAllowedError(absPath)
-                : RestrictedPathError("access to path '%s' is forbidden", absPath);
-    }
-
-    bool isAllowed(const CanonPath & absPath)
-    {
-        if (!absPath.isWithin(root))
-            return false;
-
-        if (allowedPaths) {
-            auto p = absPath.removePrefix(root);
-            if (!p.isAllowed(*allowedPaths))
-                return false;
-        }
-
-        return true;
-    }
-
-    void allowPath(CanonPath path) override
-    {
-        if (allowedPaths)
-            allowedPaths->insert(std::move(path));
-    }
-
-    bool hasAccessControl() override
-    {
-        return (bool) allowedPaths;
-    }
-
     std::optional<CanonPath> getPhysicalPath(const CanonPath & path) override
     {
         return makeAbsPath(path);
     }
 };
 
-ref<FSInputAccessor> makeFSInputAccessor(
-    const CanonPath & root,
-    std::optional<std::set<CanonPath>> && allowedPaths,
-    MakeNotAllowedError && makeNotAllowedError)
+ref<InputAccessor> makeFSInputAccessor(const CanonPath & root)
 {
-    return make_ref<FSInputAccessorImpl>(root, std::move(allowedPaths), std::move(makeNotAllowedError));
+    return make_ref<FSInputAccessor>(root);
 }
 
-ref<FSInputAccessor> makeStorePathAccessor(
+ref<InputAccessor> makeStorePathAccessor(
     ref<Store> store,
-    const StorePath & storePath,
-    MakeNotAllowedError && makeNotAllowedError)
+    const StorePath & storePath)
 {
-    return makeFSInputAccessor(CanonPath(store->toRealPath(storePath)), {}, std::move(makeNotAllowedError));
+    return makeFSInputAccessor(CanonPath(store->toRealPath(storePath)));
 }
 
 SourcePath getUnfilteredRootPath(CanonPath path)
diff --git a/src/libfetchers/fs-input-accessor.hh b/src/libfetchers/fs-input-accessor.hh
index 19a5211c8..ba5af5887 100644
--- a/src/libfetchers/fs-input-accessor.hh
+++ b/src/libfetchers/fs-input-accessor.hh
@@ -7,26 +7,12 @@ namespace nix {
 class StorePath;
 class Store;
 
-struct FSInputAccessor : InputAccessor
-{
-    virtual void checkAllowed(const CanonPath & absPath) = 0;
+ref<InputAccessor> makeFSInputAccessor(
+    const CanonPath & root);
 
-    virtual void allowPath(CanonPath path) = 0;
-
-    virtual bool hasAccessControl() = 0;
-};
-
-typedef std::function<RestrictedPathError(const CanonPath & path)> MakeNotAllowedError;
-
-ref<FSInputAccessor> makeFSInputAccessor(
-    const CanonPath & root,
-    std::optional<std::set<CanonPath>> && allowedPaths = {},
-    MakeNotAllowedError && makeNotAllowedError = {});
-
-ref<FSInputAccessor> makeStorePathAccessor(
+ref<InputAccessor> makeStorePathAccessor(
     ref<Store> store,
-    const StorePath & storePath,
-    MakeNotAllowedError && makeNotAllowedError = {});
+    const StorePath & storePath);
 
 SourcePath getUnfilteredRootPath(CanonPath path);
 
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 8cd74057c..ff4b1e823 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -9,6 +9,7 @@
 #include "processes.hh"
 #include "git.hh"
 #include "fs-input-accessor.hh"
+#include "filtering-input-accessor.hh"
 #include "mounted-input-accessor.hh"
 #include "git-utils.hh"
 #include "logging.hh"
@@ -639,7 +640,10 @@ struct GitInputScheme : InputScheme
                 repoInfo.workdirInfo.files.insert(submodule.path);
 
         ref<InputAccessor> accessor =
-            makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url));
+            AllowListInputAccessor::create(
+                makeFSInputAccessor(CanonPath(repoInfo.url)),
+                std::move(repoInfo.workdirInfo.files),
+                makeNotAllowedError(repoInfo.url));
 
         /* If the repo has submodules, return a mounted input accessor
            consisting of the accessor for the top-level repo and the

From cab41025d85d3b02f5175cf7ca2611c7a44c2cdd Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Thu, 30 Nov 2023 23:04:05 +0100
Subject: [PATCH 114/421] mention renaming of `nix doctor`

---
 doc/manual/rl-next/nix-config-show.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/rl-next/nix-config-show.md b/doc/manual/rl-next/nix-config-show.md
index 08ad207cb..b2ad3c666 100644
--- a/doc/manual/rl-next/nix-config-show.md
+++ b/doc/manual/rl-next/nix-config-show.md
@@ -3,6 +3,6 @@ issues: #7672
 prs: #9477
 description: {
 
-`nix show-config` was renamed to `nix config show` to be more consistent with the rest of the command-line interface.
+`nix show-config` was renamed to `nix config show`, and `nix doctor` was renamed to `nix config check`, to be more consistent with the rest of the command-line interface.
 
 }

From 39de819edaef2dc3e308a490bbb2b1622a932771 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Mon, 9 Oct 2023 10:08:22 +0200
Subject: [PATCH 115/421] rename debugging helper environment variable

---
 src/libutil/error.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 8488e7e21..72c346cb5 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -159,11 +159,11 @@ static std::string indent(std::string_view indentFirst, std::string_view indentR
 /**
  * A development aid for finding missing positions, to improve error messages. Example use:
  *
- *     NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 make tests/lang.sh.test
+ *     _NIX_EVAL_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 make tests/lang.sh.test
  *     git diff -U20 tests
  *
  */
-static bool printUnknownLocations = getEnv("_NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS").has_value();
+static bool printUnknownLocations = getEnv("_NIX_EVAL_SHOW_UNKNOWN_LOCATIONS").has_value();
 
 /**
  * Print a position, if it is known.

From 0301b8fc7354a94dc03b57f796bfa6e853758af8 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Thu, 30 Nov 2023 21:59:08 +0100
Subject: [PATCH 116/421] reword the experimental feature notice

- put the highlight box around all the relevant instructions
- simplify the wording
- make the link more prominent by using the whole phrase for the link text
---
 doc/manual/generate-settings.nix   | 21 ++++++-------
 doc/manual/generate-store-info.nix | 48 +++++++++++++++---------------
 2 files changed, 35 insertions(+), 34 deletions(-)

diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix
index 3add10075..74446b70b 100644
--- a/doc/manual/generate-settings.nix
+++ b/doc/manual/generate-settings.nix
@@ -31,18 +31,19 @@ let
 
       experimentalFeatureNote = optionalString (experimentalFeature != null) ''
           > **Warning**
+          >
           > This setting is part of an
           > [experimental feature](@docroot@/contributing/experimental-features.md).
-
-          To change this setting, you need to make sure the corresponding experimental feature,
-          [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}),
-          is enabled.
-          For example, include the following in [`nix.conf`](#):
-
-          ```
-          extra-experimental-features = ${experimentalFeature}
-          ${setting} = ...
-          ```
+          >
+          > To change this setting, make sure the
+          > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature})
+          > is enabled.
+          > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md):
+          >
+          > ```
+          > extra-experimental-features = ${experimentalFeature}
+          > ${setting} = ...
+          > ```
         '';
 
       showDefault = documentDefault: defaultValue:
diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix
index 73defcb71..8e26edb65 100644
--- a/doc/manual/generate-store-info.nix
+++ b/doc/manual/generate-store-info.nix
@@ -1,6 +1,6 @@
 let
   inherit (builtins) attrValues mapAttrs;
-  inherit (import <nix/utils.nix>) concatStrings optionalString;
+  inherit (import <nix/utils.nix>) concatStrings optionalString squash;
   showSettings = import <nix/generate-settings.nix>;
 in
 
@@ -10,36 +10,36 @@ let
 
   showStore = name: { settings, doc, experimentalFeature }:
     let
+      result = squash ''
+        # ${name}
 
-    result = ''
-      ## ${name}
+        ${doc}
 
-      ${doc}
+        ${experimentalFeatureNote}
 
-      ${experimentalFeatureNote}
+        ## Settings
 
-      ### Settings
-
-      ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings}
-    '';
+        ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings}
+      '';
 
       # markdown doesn't like spaces in URLs
       slug = builtins.replaceStrings [ " " ] [ "-" ] name;
 
-    experimentalFeatureNote = optionalString (experimentalFeature != null) ''
-      > **Warning**
-      > This store is part of an
-      > [experimental feature](@docroot@/contributing/experimental-features.md).
-
-      To use this store, you need to make sure the corresponding experimental feature,
-      [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}),
-      is enabled.
-      For example, include the following in [`nix.conf`](#):
-
-      ```
-      extra-experimental-features = ${experimentalFeature}
-      ```
-    '';
-  in result;
+      experimentalFeatureNote = optionalString (experimentalFeature != null) ''
+        > **Warning**
+        >
+        > This store is part of an
+        > [experimental feature](@docroot@/contributing/experimental-features.md).
+        >
+        > To use this store, make sure the
+        > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature})
+        > is enabled.
+        > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md):
+        >
+        > ```
+        > extra-experimental-features = ${experimentalFeature}
+        > ```
+      '';
+    in result;
 
 in concatStrings (attrValues (mapAttrs showStore storesInfo))

From c982198485a995d40b01b8caf62df5458046614d Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Thu, 30 Nov 2023 22:48:44 +0000
Subject: [PATCH 117/421] First step

---
 binary-tarball.nix |  81 ++++++++++++++
 flake.nix          | 262 +++++++++++++--------------------------------
 lowdown.nix        |  22 ++++
 package.nix        | 239 +++++++++++++++++++++++++++++++++++++++++
 4 files changed, 415 insertions(+), 189 deletions(-)
 create mode 100644 binary-tarball.nix
 create mode 100644 lowdown.nix
 create mode 100644 package.nix

diff --git a/binary-tarball.nix b/binary-tarball.nix
new file mode 100644
index 000000000..1fa185519
--- /dev/null
+++ b/binary-tarball.nix
@@ -0,0 +1,81 @@
+{ runCommand
+, version
+, system
+, nix
+, cacert
+}:
+
+let
+
+  installerClosureInfo = buildPackages.closureInfo {
+    rootPaths = [ nix cacert ];
+  };
+
+  env = {
+    meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
+  };
+
+in
+
+runCommand "nix-binary-tarball-${version}" env ''
+  cp ${installerClosureInfo}/registration $TMPDIR/reginfo
+  cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
+  substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
+    --subst-var-by nix ${nix} \
+    --subst-var-by cacert ${cacert}
+
+  substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
+    --subst-var-by nix ${nix} \
+    --subst-var-by cacert ${cacert}
+  substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
+    --subst-var-by nix ${nix} \
+    --subst-var-by cacert ${cacert}
+  substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
+    --subst-var-by nix ${nix} \
+    --subst-var-by cacert ${cacert}
+
+  if type -p shellcheck; then
+    # SC1090: Don't worry about not being able to find
+    #         $nix/etc/profile.d/nix.sh
+    shellcheck --exclude SC1090 $TMPDIR/install
+    shellcheck $TMPDIR/create-darwin-volume.sh
+    shellcheck $TMPDIR/install-darwin-multi-user.sh
+    shellcheck $TMPDIR/install-systemd-multi-user.sh
+
+    # SC1091: Don't panic about not being able to source
+    #         /etc/profile
+    # SC2002: Ignore "useless cat" "error", when loading
+    #         .reginfo, as the cat is a much cleaner
+    #         implementation, even though it is "useless"
+    # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
+    #         root's home directory
+    shellcheck --external-sources \
+      --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
+  fi
+
+  chmod +x $TMPDIR/install
+  chmod +x $TMPDIR/create-darwin-volume.sh
+  chmod +x $TMPDIR/install-darwin-multi-user.sh
+  chmod +x $TMPDIR/install-systemd-multi-user.sh
+  chmod +x $TMPDIR/install-multi-user
+  dir=nix-${version}-${system}
+  fn=$out/$dir.tar.xz
+  mkdir -p $out/nix-support
+  echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
+  tar cvfJ $fn \
+    --owner=0 --group=0 --mode=u+rw,uga+r \
+    --mtime='1970-01-01' \
+    --absolute-names \
+    --hard-dereference \
+    --transform "s,$TMPDIR/install,$dir/install," \
+    --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
+    --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
+    --transform "s,$NIX_STORE,$dir/store,S" \
+    $TMPDIR/install \
+    $TMPDIR/create-darwin-volume.sh \
+    $TMPDIR/install-darwin-multi-user.sh \
+    $TMPDIR/install-systemd-multi-user.sh \
+    $TMPDIR/install-multi-user \
+    $TMPDIR/reginfo \
+    $(cat ${installerClosureInfo}/store-paths)
+''
diff --git a/flake.nix b/flake.nix
index 33673575b..a1fc1cd1c 100644
--- a/flake.nix
+++ b/flake.nix
@@ -7,7 +7,7 @@
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
   inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; };
 
-  outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, flake-compat, libgit2 }:
+  outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, libgit2 }:
 
     let
       inherit (nixpkgs) lib;
@@ -34,7 +34,14 @@
         "x86_64-freebsd13" "x86_64-netbsd"
       ];
 
-      stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ];
+      stdenvs = [
+        "ccacheStdenv"
+        "clang11Stdenv"
+        "clangStdenv"
+        "gccStdenv"
+        "libcxxStdenv"
+        "stdenv"
+      ];
 
       forAllSystems = lib.genAttrs systems;
 
@@ -326,82 +333,18 @@
         '';
       };
 
-      binaryTarball = nix: pkgs:
-        let
-          inherit (pkgs) buildPackages;
-          inherit (pkgs) cacert;
-          installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix cacert ]; };
-        in
-
-        buildPackages.runCommand "nix-binary-tarball-${version}"
-          { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
-            meta.description = "Distribution-independent Nix bootstrap binaries for ${pkgs.system}";
-          }
-          ''
-            cp ${installerClosureInfo}/registration $TMPDIR/reginfo
-            cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
-            substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
-              --subst-var-by nix ${nix} \
-              --subst-var-by cacert ${cacert}
-
-            substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
-              --subst-var-by nix ${nix} \
-              --subst-var-by cacert ${cacert}
-            substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
-              --subst-var-by nix ${nix} \
-              --subst-var-by cacert ${cacert}
-            substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
-              --subst-var-by nix ${nix} \
-              --subst-var-by cacert ${cacert}
-
-            if type -p shellcheck; then
-              # SC1090: Don't worry about not being able to find
-              #         $nix/etc/profile.d/nix.sh
-              shellcheck --exclude SC1090 $TMPDIR/install
-              shellcheck $TMPDIR/create-darwin-volume.sh
-              shellcheck $TMPDIR/install-darwin-multi-user.sh
-              shellcheck $TMPDIR/install-systemd-multi-user.sh
-
-              # SC1091: Don't panic about not being able to source
-              #         /etc/profile
-              # SC2002: Ignore "useless cat" "error", when loading
-              #         .reginfo, as the cat is a much cleaner
-              #         implementation, even though it is "useless"
-              # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
-              #         root's home directory
-              shellcheck --external-sources \
-                --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
-            fi
-
-            chmod +x $TMPDIR/install
-            chmod +x $TMPDIR/create-darwin-volume.sh
-            chmod +x $TMPDIR/install-darwin-multi-user.sh
-            chmod +x $TMPDIR/install-systemd-multi-user.sh
-            chmod +x $TMPDIR/install-multi-user
-            dir=nix-${version}-${pkgs.system}
-            fn=$out/$dir.tar.xz
-            mkdir -p $out/nix-support
-            echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
-            tar cvfJ $fn \
-              --owner=0 --group=0 --mode=u+rw,uga+r \
-              --mtime='1970-01-01' \
-              --absolute-names \
-              --hard-dereference \
-              --transform "s,$TMPDIR/install,$dir/install," \
-              --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
-              --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
-              --transform "s,$NIX_STORE,$dir/store,S" \
-              $TMPDIR/install \
-              $TMPDIR/create-darwin-volume.sh \
-              $TMPDIR/install-darwin-multi-user.sh \
-              $TMPDIR/install-systemd-multi-user.sh \
-              $TMPDIR/install-multi-user \
-              $TMPDIR/reginfo \
-              $(cat ${installerClosureInfo}/store-paths)
-          '';
+      binaryTarball = nix: pkgs: pkgs.callPackage ./binary-tarball.nix {
+        inherit nix;
+      };
 
       overlayFor = getStdenv: final: prev:
-        let currentStdenv = getStdenv final; in
+        let
+          stdenv = getStdenv final;
+
+          lowdown-nix = final.callPackage ./lowdown.nix {
+            inherit lowdown-src stdenv;
+          };
+        in
         {
           nixStable = prev.nix;
 
@@ -409,129 +352,70 @@
           nixUnstable = prev.nixUnstable;
 
           nix =
-          with final;
-          with commonDeps {
-            inherit pkgs;
-            inherit (currentStdenv.hostPlatform) isStatic;
-          };
-          let
-            canRunInstalled = currentStdenv.buildPlatform.canExecute currentStdenv.hostPlatform;
-          in currentStdenv.mkDerivation (finalAttrs: {
-            name = "nix-${version}";
-            inherit version;
+            let
+              officialRelease = false;
+              versionSuffix =
+                if officialRelease
+                then ""
+                else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";
 
-            src = nixSrc;
-            VERSION_SUFFIX = versionSuffix;
+              sh = final.busybox-sandbox-shell or (final.busybox.override {
+                useMusl = true;
+                enableStatic = true;
+                enableMinimal = true;
+                extraConfig = ''
+                  CONFIG_FEATURE_FANCY_ECHO y
+                  CONFIG_FEATURE_SH_MATH y
+                  CONFIG_FEATURE_SH_MATH_64 y
 
-            outputs = [ "out" "dev" "doc" ]
-              ++ lib.optional (currentStdenv.hostPlatform != currentStdenv.buildPlatform) "check";
+                  CONFIG_ASH y
+                  CONFIG_ASH_OPTIMIZE_FOR_SIZE y
 
-            nativeBuildInputs = nativeBuildDeps;
-            buildInputs = buildDeps
-              # There have been issues building these dependencies
-              ++ lib.optionals (currentStdenv.hostPlatform == currentStdenv.buildPlatform) awsDeps
-              ++ lib.optionals finalAttrs.doCheck checkDeps;
+                  CONFIG_ASH_ALIAS y
+                  CONFIG_ASH_BASH_COMPAT y
+                  CONFIG_ASH_CMDCMD y
+                  CONFIG_ASH_ECHO y
+                  CONFIG_ASH_GETOPTS y
+                  CONFIG_ASH_INTERNAL_GLOB y
+                  CONFIG_ASH_JOB_CONTROL y
+                  CONFIG_ASH_PRINTF y
+                  CONFIG_ASH_TEST y
+                '';
+              });
 
-            propagatedBuildInputs = propagatedDeps;
+              boehmgc = (final.boehmgc.override {
+                enableLargeConfig = true;
+              }).overrideAttrs(o: {
+                patches = (o.patches or []) ++ [
+                  ./boehmgc-coroutine-sp-fallback.diff
 
-            disallowedReferences = [ boost-nix ];
+                  # https://github.com/ivmai/bdwgc/pull/586
+                  ./boehmgc-traceable_allocator-public.diff
+                ];
+              });
 
-            preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic)
-              ''
-                # Copy libboost_context so we don't get all of Boost in our closure.
-                # https://github.com/NixOS/nixpkgs/issues/45462
-                mkdir -p $out/lib
-                cp -pd ${boost-nix}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib
-                rm -f $out/lib/*.a
-                ${lib.optionalString currentStdenv.hostPlatform.isLinux ''
-                  chmod u+w $out/lib/*.so.*
-                  patchelf --set-rpath $out/lib:${currentStdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
-                ''}
-                ${lib.optionalString currentStdenv.hostPlatform.isDarwin ''
-                  for LIB in $out/lib/*.dylib; do
-                    chmod u+w $LIB
-                    install_name_tool -id $LIB $LIB
-                    install_name_tool -delete_rpath ${boost-nix}/lib/ $LIB || true
-                  done
-                  install_name_tool -change ${boost-nix}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
-                ''}
-              '';
-
-            configureFlags = configureFlags ++
-              [ "--sysconfdir=/etc" ] ++
-              lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" ++
-              [ (lib.enableFeature finalAttrs.doCheck "tests") ] ++
-              lib.optionals finalAttrs.doCheck testConfigureFlags ++
-              lib.optional (!canRunInstalled) "--disable-doc-gen";
-
-            enableParallelBuilding = true;
-
-            makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1";
-
-            doCheck = true;
-
-            installFlags = "sysconfdir=$(out)/etc";
-
-            postInstall = ''
-              mkdir -p $doc/nix-support
-              echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
-              ${lib.optionalString currentStdenv.hostPlatform.isStatic ''
-              mkdir -p $out/nix-support
-              echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
-              ''}
-              ${lib.optionalString currentStdenv.isDarwin ''
-              install_name_tool \
-                -change ${boost-nix}/lib/libboost_context.dylib \
-                $out/lib/libboost_context.dylib \
-                $out/lib/libnixutil.dylib
-              install_name_tool \
-                -change ${boost-nix}/lib/libboost_regex.dylib \
-                $out/lib/libboost_regex.dylib \
-                $out/lib/libnixexpr.dylib
-              ''}
-            '';
-
-            doInstallCheck = finalAttrs.doCheck;
-            installCheckFlags = "sysconfdir=$(out)/etc";
-            installCheckTarget = "installcheck"; # work around buggy detection in stdenv
-
-            separateDebugInfo = !currentStdenv.hostPlatform.isStatic;
-
-            strictDeps = true;
-
-            hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-
-            passthru.perl-bindings = final.callPackage ./perl {
-              inherit fileset;
-              stdenv = currentStdenv;
+            in final.callPackage ./package.nix {
+              inherit
+                boehmgc
+                fileset
+                sh
+                stdenv
+                versionSuffix
+                ;
+              boost = final.boost.override { enableIcu = false; };
+              libgit2 = final.libgit2.overrideAttrs (attrs: {
+                src = libgit2;
+                version = libgit2.lastModifiedDate;
+                cmakeFlags = attrs.cmakeFlags or []
+                  ++ [ "-DUSE_SSH=exec" ];
+              });
+              lowdown = lowdown-nix;
+              officialRelease = false;
             };
 
-            meta.platforms = lib.platforms.unix;
-            meta.mainProgram = "nix";
-          });
-
-          boost-nix = final.boost.override {
-            enableIcu = false;
+            inherit lowdown-nix;
           };
 
-          lowdown-nix = with final; currentStdenv.mkDerivation rec {
-            name = "lowdown-0.9.0";
-
-            src = lowdown-src;
-
-            outputs = [ "out" "bin" "dev" ];
-
-            nativeBuildInputs = [ buildPackages.which ];
-
-            configurePhase = ''
-                ${if (currentStdenv.isDarwin && currentStdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""}
-                ./configure \
-                  PREFIX=${placeholder "dev"} \
-                  BINDIR=${placeholder "bin"}/bin
-            '';
-          };
-        };
-
     in {
       # A Nixpkgs overlay that overrides the 'nix' and
       # 'nix.perl-bindings' packages.
diff --git a/lowdown.nix b/lowdown.nix
new file mode 100644
index 000000000..5f469fad5
--- /dev/null
+++ b/lowdown.nix
@@ -0,0 +1,22 @@
+{ lib
+, stdenv
+, which
+, lowdown-src
+}:
+
+stdenv.mkDerivation rec {
+  name = "lowdown-0.9.0";
+
+  src = lowdown-src;
+
+  outputs = [ "out" "bin" "dev" ];
+
+  nativeBuildInputs = [ which ];
+
+  configurePhase = ''
+      ${lib.optionalString (stdenv.isDarwin && stdenv.isAarch64) "echo \"HAVE_SANDBOX_INIT=false\" > configure.local"}
+      ./configure \
+        PREFIX=${placeholder "dev"} \
+        BINDIR=${placeholder "bin"}/bin
+  '';
+}
diff --git a/package.nix b/package.nix
new file mode 100644
index 000000000..ae075acf7
--- /dev/null
+++ b/package.nix
@@ -0,0 +1,239 @@
+{ lib
+, callPackage
+, stdenv
+, versionSuffix ? ""
+, officialRelease ? false
+, buildUnreleasedNotes ? false
+, autoconf-archive
+, autoreconfHook
+, aws-sdk-cpp
+, boehmgc
+, nlohmann_json
+, bison
+, boost
+, brotli
+, bzip2
+, changelog-d
+, curl
+, editline
+, fileset
+, flex
+, git
+, gtest
+, jq
+, libarchive
+, libcpuid
+, libgit2
+, libseccomp
+, libsodium
+, lowdown
+, mdbook
+, mdbook-linkcheck
+, mercurial
+, openssh
+, openssl
+, pkg-config
+, rapidcheck
+, sh
+, sqlite
+, util-linux
+, xz
+}:
+
+let
+
+  version = lib.fileContents ./.version + versionSuffix;
+
+  inherit (stdenv.hostPlatform) isStatic;
+
+  canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
+in
+
+stdenv.mkDerivation (finalAttrs: {
+  name = "nix-${version}";
+  inherit version;
+
+  src =
+    let
+      baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.;
+      configureFiles = fileset.unions [
+        ./.version
+        ./configure.ac
+        ./m4
+        # TODO: do we really need README.md? It doesn't seem used in the build.
+        ./README.md
+      ];
+
+      topLevelBuildFiles = fileset.unions [
+        ./local.mk
+        ./Makefile
+        ./Makefile.config.in
+        ./mk
+      ];
+
+      functionalTestFiles = fileset.unions [
+        ./tests/functional
+        (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
+      ];
+
+    in
+      fileset.toSource {
+        root = ./.;
+        fileset = fileset.intersect baseFiles (fileset.unions [
+          configureFiles
+          topLevelBuildFiles
+          ./boehmgc-coroutine-sp-fallback.diff
+          ./doc
+          ./misc
+          ./precompiled-headers.h
+          ./src
+          ./unit-test-data
+          ./COPYING
+          ./scripts/local.mk
+          functionalTestFiles
+        ]);
+      };
+
+  VERSION_SUFFIX = versionSuffix;
+
+  outputs = [ "out" "dev" "doc" ]
+    ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "check";
+
+  nativeBuildInputs = [
+    bison
+    flex
+    (lib.getBin lowdown)
+    mdbook
+    mdbook-linkcheck
+    autoconf-archive
+    autoreconfHook
+    pkg-config
+
+    # Tests
+    git
+    mercurial # FIXME: remove? only needed for tests
+    jq # Also for custom mdBook preprocessor.
+    openssh # only needed for tests (ssh-keygen)
+  ]
+  ++ lib.optional stdenv.hostPlatform.isLinux util-linux
+  # Official releases don't have rl-next, so we don't need to compile a changelog
+  ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d;
+
+  buildInputs = [
+    boost
+    brotli
+    bzip2
+    curl
+    editline
+    libarchive
+    libgit2
+    libsodium
+    lowdown
+    openssl
+    sqlite
+    xz
+  ]
+  ++ lib.optionals stdenv.isLinux [libseccomp]
+  ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid
+  # There have been issues building these dependencies
+  ++ lib.optionals (stdenv.hostPlatform == stdenv.buildPlatform) (lib.optional (stdenv.isLinux || stdenv.isDarwin)
+    (aws-sdk-cpp.override {
+      apis = ["s3" "transfer"];
+      customMemoryManagement = false;
+    }))
+  ++ lib.optionals finalAttrs.doCheck ([
+    gtest
+    rapidcheck
+  ]);
+
+  propagatedBuildInputs = [
+    boehmgc
+    nlohmann_json
+  ];
+
+  disallowedReferences = [ boost ];
+
+  preConfigure = lib.optionalString (! stdenv.hostPlatform.isStatic)
+    ''
+      # Copy libboost_context so we don't get all of Boost in our closure.
+      # https://github.com/NixOS/nixpkgs/issues/45462
+      mkdir -p $out/lib
+      cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib
+      rm -f $out/lib/*.a
+      ${lib.optionalString stdenv.hostPlatform.isLinux ''
+        chmod u+w $out/lib/*.so.*
+        patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
+      ''}
+      ${lib.optionalString stdenv.hostPlatform.isDarwin ''
+        for LIB in $out/lib/*.dylib; do
+          chmod u+w $LIB
+          install_name_tool -id $LIB $LIB
+          install_name_tool -delete_rpath ${boost}/lib/ $LIB || true
+        done
+        install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
+      ''}
+    '';
+
+  configureFlags =
+    lib.optionals stdenv.isLinux [
+      "--with-boost=${boost}/lib"
+      "--with-sandbox-shell=${sh}/bin/busybox"
+    ]
+    ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [
+      "LDFLAGS=-fuse-ld=gold"
+    ]
+    ++ [ "--sysconfdir=/etc" ]
+    ++ lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell"
+    ++ [ (lib.enableFeature finalAttrs.doCheck "tests") ]
+    ++ lib.optionals finalAttrs.doCheck ([ "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" ]
+    ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
+      "--enable-install-unit-tests"
+      "--with-check-bin-dir=${builtins.placeholder "check"}/bin"
+      "--with-check-lib-dir=${builtins.placeholder "check"}/lib"
+    ])
+    ++ lib.optional (!canRunInstalled) "--disable-doc-gen";
+
+  enableParallelBuilding = true;
+
+  makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1";
+
+  doCheck = true;
+
+  installFlags = "sysconfdir=$(out)/etc";
+
+  postInstall = ''
+    mkdir -p $doc/nix-support
+    echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
+    ${lib.optionalString stdenv.hostPlatform.isStatic ''
+      mkdir -p $out/nix-support
+      echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
+    ''}
+    ${lib.optionalString stdenv.isDarwin ''
+      install_name_tool \
+      -change ${boost}/lib/libboost_context.dylib \
+      $out/lib/libboost_context.dylib \
+      $out/lib/libnixutil.dylib
+      install_name_tool \
+      -change ${boost}/lib/libboost_regex.dylib \
+      $out/lib/libboost_regex.dylib \
+      $out/lib/libnixexpr.dylib
+    ''}
+  '';
+
+  doInstallCheck = finalAttrs.doCheck;
+  installCheckFlags = "sysconfdir=$(out)/etc";
+  installCheckTarget = "installcheck"; # work around buggy detection in stdenv
+
+  separateDebugInfo = !stdenv.hostPlatform.isStatic;
+
+  strictDeps = true;
+
+  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+
+  passthru.perl-bindings = callPackage ./perl {
+    inherit fileset stdenv;
+  };
+
+  meta.platforms = lib.platforms.unix;
+  meta.mainProgram = "nix";
+})

From c64190e65048547712fcf7a0ae09fbfd0a709474 Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Thu, 30 Nov 2023 22:49:02 +0000
Subject: [PATCH 118/421] Run statix

---
 flake.nix   | 6 +++---
 package.nix | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/flake.nix b/flake.nix
index a1fc1cd1c..e32a84ae5 100644
--- a/flake.nix
+++ b/flake.nix
@@ -7,7 +7,7 @@
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
   inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; };
 
-  outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, libgit2 }:
+  outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, libgit2, ... }:
 
     let
       inherit (nixpkgs) lib;
@@ -183,7 +183,7 @@
           "--enable-internal-api-docs"
         ];
 
-        changelog-d = pkgs.buildPackages.changelog-d;
+        inherit (pkgs.buildPackages) changelog-d;
 
         nativeBuildDeps =
           [
@@ -349,7 +349,7 @@
           nixStable = prev.nix;
 
           # Forward from the previous stage as we don’t want it to pick the lowdown override
-          nixUnstable = prev.nixUnstable;
+          inherit (prev) nixUnstable;
 
           nix =
             let
diff --git a/package.nix b/package.nix
index ae075acf7..8d62120fb 100644
--- a/package.nix
+++ b/package.nix
@@ -141,10 +141,10 @@ stdenv.mkDerivation (finalAttrs: {
       apis = ["s3" "transfer"];
       customMemoryManagement = false;
     }))
-  ++ lib.optionals finalAttrs.doCheck ([
+  ++ lib.optionals finalAttrs.doCheck [
     gtest
     rapidcheck
-  ]);
+  ];
 
   propagatedBuildInputs = [
     boehmgc

From f55ee7cf7753caee7a27052fab679d8c8fe27cc4 Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Thu, 30 Nov 2023 22:53:07 +0000
Subject: [PATCH 119/421] little refactoring

---
 flake.nix | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/flake.nix b/flake.nix
index e32a84ae5..544a07ba6 100644
--- a/flake.nix
+++ b/flake.nix
@@ -459,8 +459,21 @@
         # to https://nixos.org/nix/install. It downloads the binary
         # tarball for the user's system and calls the second half of the
         # installation script.
-        installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ];
-        installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"];
+        installerScript = installScriptFor [
+          "aarch64-linux"
+          "armv6l-linux"
+          "armv7l-linux"
+          "i686-linux"
+          "x86_64-linux"
+          "aarch64-darwin"
+          "x86_64-darwin"
+        ];
+        installerScriptForGHA = installScriptFor [
+          "armv6l-linux"
+          "armv7l-linux"
+          "x86_64-linux"
+          "x86_64-darwin"
+        ];
 
         # docker image with Nix inside
         dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage);

From 02d9cf2d303e4e7e283dba2f3181f3e40843c354 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 1 Dec 2023 00:41:19 +0100
Subject: [PATCH 120/421] shorten the quick start chapter

this focuses on `nix-shell -p` and refers to search.nixos.org for
package search, which is currently the easiest and most effective way to
find program names.
---
 doc/manual/src/quick-start.md | 87 +++++++----------------------------
 1 file changed, 16 insertions(+), 71 deletions(-)

diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md
index 5f54abbde..04a0b7c96 100644
--- a/doc/manual/src/quick-start.md
+++ b/doc/manual/src/quick-start.md
@@ -1,10 +1,9 @@
 # Quick Start
 
-This chapter is for impatient people who don't like reading
-documentation.  For more in-depth information you are kindly referred
-to subsequent chapters.
+This chapter is for impatient people who don't like reading documentation.
+For more in-depth information you are kindly referred to subsequent chapters.
 
-1. Install Nix by running the following:
+1. Install Nix:
 
    ```console
    $ curl -L https://nixos.org/nix/install | sh
@@ -13,87 +12,33 @@ to subsequent chapters.
    The install script will use `sudo`, so make sure you have sufficient rights.
    On Linux, `--daemon` can be omitted for a single-user install.
 
-   For other installation methods, see [here](installation/index.md).
+   For other installation methods, see the detailed [installation instructions](installation/index.md).
 
-1. See what installable packages are currently available in the
-   channel:
+1. Run software without installing it permanently:
 
    ```console
-   $ nix-env --query --available --attr-path
-   nixpkgs.docbook_xml_dtd_43                    docbook-xml-4.3
-   nixpkgs.docbook_xml_dtd_45                    docbook-xml-4.5
-   nixpkgs.firefox                               firefox-33.0.2
-   nixpkgs.hello                                 hello-2.9
-   nixpkgs.libxslt                               libxslt-1.1.28
-   …
+   $ nix-shell --packages cowsay lolcat
    ```
 
-1. Install some packages from the channel:
+   This downloads the specified packages with all their dependencies, and drops you into a Bash shell where the commands provided by those packages are present.
+   This will not affect your normal environment:
 
    ```console
-   $ nix-env --install --attr nixpkgs.hello
+   [nix-shell:~]$ cowsay Hello, Nix! | lolcat
    ```
 
-   This should download pre-built packages; it should not build them
-   locally (if it does, something went wrong).
-
-1. Test that they work:
+   Exiting the shell will make the programs disappear again:
 
    ```console
-   $ which hello
-   /home/eelco/.nix-profile/bin/hello
-   $ hello
-   Hello, world!
-   ```
-
-1. Uninstall a package:
-
-   ```console
-   $ nix-env --uninstall hello
-   ```
-
-1. You can also test a package without installing it:
-
-   ```console
-   $ nix-shell --packages hello
-   ```
-
-   This builds or downloads GNU Hello and its dependencies, then drops
-   you into a Bash shell where the `hello` command is present, all
-   without affecting your normal environment:
-
-   ```console
-   [nix-shell:~]$ hello
-   Hello, world!
-
    [nix-shell:~]$ exit
-
-   $ hello
-   hello: command not found
+   $ lolcat
+   lolcat: command not found
    ```
 
-1. To keep up-to-date with the channel, do:
+1. Search for more packages on <search.nixos.org> to try them out.
+
+1. Free up storage space:
 
    ```console
-   $ nix-channel --update nixpkgs
-   $ nix-env --upgrade '*'
-   ```
-
-   The latter command will upgrade each installed package for which
-   there is a “newer” version (as determined by comparing the version
-   numbers).
-
-1. If you're unhappy with the result of a `nix-env` action (e.g., an
-   upgraded package turned out not to work properly), you can go back:
-
-   ```console
-   $ nix-env --rollback
-   ```
-
-1. You should periodically run the Nix garbage collector to get rid of
-   unused packages, since uninstalls or upgrades don't actually delete
-   them:
-
-   ```console
-   $ nix-collect-garbage --delete-old
+   $ nix-collect-garbage
    ```

From 908a011a4a2fe4e494e5b6e4c94f013f159f3616 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <github@infinisil.com>
Date: Fri, 1 Dec 2023 00:50:20 +0100
Subject: [PATCH 121/421] Revert "Switch from std::regex to boost::regex"

---
 flake.nix              | 22 +++++++---------------
 src/libexpr/local.mk   |  2 +-
 src/libexpr/primops.cc | 35 ++++++++++++-----------------------
 3 files changed, 20 insertions(+), 39 deletions(-)

diff --git a/flake.nix b/flake.nix
index 33673575b..822b3d31e 100644
--- a/flake.nix
+++ b/flake.nix
@@ -157,7 +157,7 @@
 
         configureFlags =
           lib.optionals stdenv.isLinux [
-            "--with-boost=${boost-nix}/lib"
+            "--with-boost=${boost}/lib"
             "--with-sandbox-shell=${sh}/bin/busybox"
           ]
           ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [
@@ -210,7 +210,7 @@
               version = libgit2.lastModifiedDate;
               cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"];
             }))
-            boost-nix
+            boost
             lowdown-nix
             libsodium
           ]
@@ -434,14 +434,14 @@
 
             propagatedBuildInputs = propagatedDeps;
 
-            disallowedReferences = [ boost-nix ];
+            disallowedReferences = [ boost ];
 
             preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic)
               ''
                 # Copy libboost_context so we don't get all of Boost in our closure.
                 # https://github.com/NixOS/nixpkgs/issues/45462
                 mkdir -p $out/lib
-                cp -pd ${boost-nix}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib
+                cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
                 rm -f $out/lib/*.a
                 ${lib.optionalString currentStdenv.hostPlatform.isLinux ''
                   chmod u+w $out/lib/*.so.*
@@ -451,9 +451,9 @@
                   for LIB in $out/lib/*.dylib; do
                     chmod u+w $LIB
                     install_name_tool -id $LIB $LIB
-                    install_name_tool -delete_rpath ${boost-nix}/lib/ $LIB || true
+                    install_name_tool -delete_rpath ${boost}/lib/ $LIB || true
                   done
-                  install_name_tool -change ${boost-nix}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
+                  install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
                 ''}
               '';
 
@@ -481,13 +481,9 @@
               ''}
               ${lib.optionalString currentStdenv.isDarwin ''
               install_name_tool \
-                -change ${boost-nix}/lib/libboost_context.dylib \
+                -change ${boost}/lib/libboost_context.dylib \
                 $out/lib/libboost_context.dylib \
                 $out/lib/libnixutil.dylib
-              install_name_tool \
-                -change ${boost-nix}/lib/libboost_regex.dylib \
-                $out/lib/libboost_regex.dylib \
-                $out/lib/libnixexpr.dylib
               ''}
             '';
 
@@ -510,10 +506,6 @@
             meta.mainProgram = "nix";
           });
 
-          boost-nix = final.boost.override {
-            enableIcu = false;
-          };
-
           lowdown-nix = with final; currentStdenv.mkDerivation rec {
             name = "lowdown-0.9.0";
 
diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index c07a18bb5..ed6bc761a 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -16,7 +16,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib
 
 libexpr_LIBS = libutil libstore libfetchers
 
-libexpr_LDFLAGS += -lboost_context -lboost_regex -pthread
+libexpr_LDFLAGS += -lboost_context -pthread
 ifdef HOST_LINUX
  libexpr_LDFLAGS += -ldl
 endif
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index ebf2549e4..146a7603c 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -17,7 +17,6 @@
 #include "primops.hh"
 
 #include <boost/container/small_vector.hpp>
-#include <boost/regex.hpp>
 #include <nlohmann/json.hpp>
 
 #include <sys/types.h>
@@ -26,6 +25,7 @@
 
 #include <algorithm>
 #include <cstring>
+#include <regex>
 #include <dlfcn.h>
 
 #include <cmath>
@@ -3886,30 +3886,19 @@ static RegisterPrimOp primop_convertHash({
     .fun = prim_convertHash,
 });
 
-// regex aliases, switch between boost and std
-using regex = boost::regex;
-using regex_error = boost::regex_error;
-using cmatch = boost::cmatch;
-using cregex_iterator = boost::cregex_iterator;
-namespace regex_constants = boost::regex_constants;
-// overloaded function alias
-constexpr auto regex_match = [] (auto &&...args) {
-    return boost::regex_match(std::forward<decltype(args)>(args)...);
- };
-
 struct RegexCache
 {
     // TODO use C++20 transparent comparison when available
-    std::unordered_map<std::string_view, regex> cache;
+    std::unordered_map<std::string_view, std::regex> cache;
     std::list<std::string> keys;
 
-    regex get(std::string_view re)
+    std::regex get(std::string_view re)
     {
         auto it = cache.find(re);
         if (it != cache.end())
             return it->second;
         keys.emplace_back(re);
-        return cache.emplace(keys.back(), regex(keys.back(), regex::extended)).first->second;
+        return cache.emplace(keys.back(), std::regex(keys.back(), std::regex::extended)).first->second;
     }
 };
 
@@ -3929,8 +3918,8 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v)
         NixStringContext context;
         const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.match");
 
-        cmatch match;
-        if (!regex_match(str.begin(), str.end(), match, regex)) {
+        std::cmatch match;
+        if (!std::regex_match(str.begin(), str.end(), match, regex)) {
             v.mkNull();
             return;
         }
@@ -3945,8 +3934,8 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v)
                 (v.listElems()[i] = state.allocValue())->mkString(match[i + 1].str());
         }
 
-    } catch (regex_error & e) {
-        if (e.code() == regex_constants::error_space) {
+    } catch (std::regex_error & e) {
+        if (e.code() == std::regex_constants::error_space) {
             // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
             state.debugThrowLastTrace(EvalError({
                 .msg = hintfmt("memory limit exceeded by regular expression '%s'", re),
@@ -4009,8 +3998,8 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v)
         NixStringContext context;
         const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.split");
 
-        auto begin = cregex_iterator(str.begin(), str.end(), regex);
-        auto end = cregex_iterator();
+        auto begin = std::cregex_iterator(str.begin(), str.end(), regex);
+        auto end = std::cregex_iterator();
 
         // Any matches results are surrounded by non-matching results.
         const size_t len = std::distance(begin, end);
@@ -4049,8 +4038,8 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 
         assert(idx == 2 * len + 1);
 
-    } catch (regex_error & e) {
-        if (e.code() == regex_constants::error_space) {
+    } catch (std::regex_error & e) {
+        if (e.code() == std::regex_constants::error_space) {
             // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
             state.debugThrowLastTrace(EvalError({
                 .msg = hintfmt("memory limit exceeded by regular expression '%s'", re),

From 4781e7fa7048d2861172baaaa04a7be4b8a2b631 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Thu, 30 Nov 2023 23:07:09 +0100
Subject: [PATCH 122/421] Document each store type on its own page

This makes for more useful manual table of contents, that displays the
information at a glance.

The `nix help-stores` command is kept as-is, even though it will show up
in the manual with the same information as these pages due to the way it
is written as a "`--help`-style" command. Deciding what to do with that
command is left for a later PR.

This change also lists all store types at the top of the respective overview page.

Co-authored-by: John Ericson <John.Ericson@Obsidian.Systems
---
 .gitignore                                   |  2 +
 doc/manual/generate-manpage.nix              | 64 ++++++++++++++------
 doc/manual/generate-store-info.nix           | 28 ++++++---
 doc/manual/generate-store-types.nix          | 39 ++++++++++++
 doc/manual/generate-xp-features.nix          |  4 +-
 doc/manual/local.mk                          | 11 +++-
 doc/manual/src/SUMMARY.md.in                 |  2 +
 doc/manual/src/contributing/cli-guideline.md |  2 +-
 doc/manual/src/store/index.md                |  2 +-
 doc/manual/src/store/types/index.md.in       |  5 +-
 doc/manual/utils.nix                         |  8 +++
 src/libcmd/common-eval-args.cc               |  2 +-
 src/libexpr/primops.cc                       |  2 +-
 src/libstore/globals.hh                      | 11 ++--
 src/libstore/remote-store.cc                 |  2 +-
 src/libstore/ssh-store-config.hh             |  2 +-
 src/libstore/store-api.hh                    |  4 +-
 src/libutil/experimental-features.cc         |  2 +-
 src/nix/nix.md                               | 10 ++-
 19 files changed, 155 insertions(+), 47 deletions(-)
 create mode 100644 doc/manual/generate-store-types.nix

diff --git a/.gitignore b/.gitignore
index bcf9c4a01..767a5d6ed 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,6 +22,8 @@ perl/Makefile.config
 /doc/manual/xp-features.json
 /doc/manual/src/SUMMARY.md
 /doc/manual/src/SUMMARY-rl-next.md
+/doc/manual/src/store/types/*
+!/doc/manual/src/store/types/index.md.in
 /doc/manual/src/command-ref/new-cli
 /doc/manual/src/command-ref/conf-file.md
 /doc/manual/src/command-ref/experimental-features-shortlist.md
diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix
index c4b9d1335..365422af7 100644
--- a/doc/manual/generate-manpage.nix
+++ b/doc/manual/generate-manpage.nix
@@ -51,7 +51,7 @@ let
 
         ${maybeSubcommands}
 
-        ${maybeStoreDocs}
+        ${maybeProse}
 
         ${maybeOptions}
       '';
@@ -91,25 +91,55 @@ let
         * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description}
       '';
 
-      # FIXME: this is a hack.
-      # store parameters should not be part of command documentation to begin
-      # with, but instead be rendered on separate pages.
-      maybeStoreDocs = optionalString (details ? doc)
-        (replaceStrings [ "@stores@" ] [ (showStoreDocs inlineHTML commandInfo.stores) ] details.doc);
+      maybeProse =
+        # FIXME: this is a horrible hack to keep `nix help-stores` working.
+        # the correct answer to this is to remove that command and replace it
+        # by statically generated manpages or the output of something like `nix
+        # store info <store type>`.
+        let
+          help-stores = ''
+            ${index}
 
-      maybeOptions = let
-        allVisibleOptions = filterAttrs
-          (_: o: ! o.hiddenCategory)
-          (details.flags // toplevel.flags);
-      in optionalString (allVisibleOptions != {}) ''
-        # Options
+            ${allStores}
+          '';
+          index = replaceStrings
+            [ "@store-types@" ] [ storesOverview ]
+            details.doc;
+          storesOverview =
+            let
+              showEntry = store:
+                "- [${store.name}](#${store.slug})";
+            in
+            concatStringsSep "\n" (map showEntry storesList) + "\n";
+          allStores = concatStringsSep "\n" (attrValues storePages);
+          storePages = listToAttrs
+            (map (s: { name = s.filename; value = s.page; }) storesList);
+          storesList = showStoreDocs {
+            storeInfo = commandInfo.stores;
+            inherit inlineHTML;
+          };
+        in
+        optionalString (details ? doc) (
+          if match "@store-types@" details.doc != [ ]
+          then help-stores
+          else details.doc
+        );
 
-        ${showOptions inlineHTML allVisibleOptions}
+      maybeOptions =
+        let
+          allVisibleOptions = filterAttrs
+            (_: o: ! o.hiddenCategory)
+            (details.flags // toplevel.flags);
+        in
+        optionalString (allVisibleOptions != { }) ''
+          # Options
 
-        > **Note**
-        >
-        > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags.
-      '';
+          ${showOptions inlineHTML allVisibleOptions}
+
+          > **Note**
+          >
+          > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags.
+        '';
 
       showOptions = inlineHTML: allOptions:
         let
diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix
index 8e26edb65..57247a181 100644
--- a/doc/manual/generate-store-info.nix
+++ b/doc/manual/generate-store-info.nix
@@ -1,14 +1,20 @@
 let
-  inherit (builtins) attrValues mapAttrs;
-  inherit (import <nix/utils.nix>) concatStrings optionalString squash;
+  inherit (builtins) attrNames listToAttrs concatStringsSep readFile replaceStrings;
+  inherit (import <nix/utils.nix>) optionalString filterAttrs trim squash toLower unique indent;
   showSettings = import <nix/generate-settings.nix>;
 in
 
-inlineHTML: storesInfo:
+{
+  # data structure describing all stores and their parameters
+  storeInfo,
+  # whether to add inline HTML tags
+  # `lowdown` does not eat those for one of the output modes
+  inlineHTML,
+}:
 
 let
 
-  showStore = name: { settings, doc, experimentalFeature }:
+  showStore = { name, slug }: { settings, doc, experimentalFeature }:
     let
       result = squash ''
         # ${name}
@@ -22,9 +28,6 @@ let
         ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings}
       '';
 
-      # markdown doesn't like spaces in URLs
-      slug = builtins.replaceStrings [ " " ] [ "-" ] name;
-
       experimentalFeatureNote = optionalString (experimentalFeature != null) ''
         > **Warning**
         >
@@ -42,4 +45,13 @@ let
       '';
     in result;
 
-in concatStrings (attrValues (mapAttrs showStore storesInfo))
+  storesList = map
+    (name: rec {
+      inherit name;
+      slug = replaceStrings [ " " ] [ "-" ] (toLower name);
+      filename = "${slug}.md";
+      page = showStore { inherit name slug; } storeInfo.${name};
+    })
+    (attrNames storeInfo);
+
+in storesList
diff --git a/doc/manual/generate-store-types.nix b/doc/manual/generate-store-types.nix
new file mode 100644
index 000000000..3b78a0e1b
--- /dev/null
+++ b/doc/manual/generate-store-types.nix
@@ -0,0 +1,39 @@
+let
+  inherit (builtins) attrNames listToAttrs concatStringsSep readFile replaceStrings;
+  showSettings = import <nix/generate-settings.nix>;
+  showStoreDocs = import <nix/generate-store-info.nix>;
+in
+
+storeInfo:
+
+let
+  storesList = showStoreDocs {
+    inherit storeInfo;
+    inlineHTML = true;
+  };
+
+  index =
+    let
+      showEntry = store:
+        "- [${store.name}](./${store.filename})";
+    in
+    concatStringsSep "\n" (map showEntry storesList);
+
+  "index.md" = replaceStrings
+    [ "@store-types@" ] [ index ]
+    (readFile ./src/store/types/index.md.in);
+
+  tableOfContents =
+    let
+      showEntry = store:
+        "    - [${store.name}](store/types/${store.filename})";
+    in
+    concatStringsSep "\n" (map showEntry storesList) + "\n";
+
+  "SUMMARY.md" = tableOfContents;
+
+  storePages = listToAttrs
+    (map (s: { name = s.filename; value = s.page; }) storesList);
+
+in
+storePages // { inherit "index.md" "SUMMARY.md"; }
diff --git a/doc/manual/generate-xp-features.nix b/doc/manual/generate-xp-features.nix
index fc7c7d4cf..0eec0e1da 100644
--- a/doc/manual/generate-xp-features.nix
+++ b/doc/manual/generate-xp-features.nix
@@ -8,4 +8,6 @@ let
 
       ${doc}
     '';
-in xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps)))
+in
+
+xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps)))
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index fa9db9f02..456000d3d 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -97,10 +97,17 @@ $(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md
 	$(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@
 	@rm $^.tmp
 
-$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
+$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
 	@cp $< $@
 	@$(call process-includes,$@,$@)
 
+$(d)/src/store/types: $(d)/nix.json $(d)/utils.nix $(d)/generate-store-info.nix  $(d)/generate-store-types.nix $(d)/src/store/types/index.md.in $(doc_nix)
+	@# FIXME: build out of tree!
+	@rm -rf $@.tmp
+	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-store-types.nix (builtins.fromJSON (builtins.readFile $<)).stores'
+	@# do not destroy existing contents
+	@mv $@.tmp/* $@/
+
 $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(doc_nix)
 	@rm -rf $@ $@.tmp
 	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)'
@@ -200,7 +207,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
 # `@docroot@` is to be preserved for documenting the mechanism
 # FIXME: maybe contributing guides should live right next to the code
 # instead of in the manual
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md
 	$(trace-gen) \
 		tmp="$$(mktemp -d)"; \
 		cp -r doc/manual "$$tmp"; \
diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in
index 686d3e8d7..c67ddc6cb 100644
--- a/doc/manual/src/SUMMARY.md.in
+++ b/doc/manual/src/SUMMARY.md.in
@@ -20,6 +20,8 @@
   - [File System Object](store/file-system-object.md)
   - [Store Object](store/store-object.md)
   - [Store Path](store/store-path.md)
+  - [Store Types](store/types/index.md)
+{{#include ./store/types/SUMMARY.md}}
 - [Nix Language](language/index.md)
   - [Data Types](language/values.md)
   - [Language Constructs](language/constructs.md)
diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/contributing/cli-guideline.md
index e53d2d178..f7e24d96b 100644
--- a/doc/manual/src/contributing/cli-guideline.md
+++ b/doc/manual/src/contributing/cli-guideline.md
@@ -426,7 +426,7 @@ This leads to the following guidelines:
 ### Examples
 
 
-This is bad, because all keys must be assumed to be store implementations:
+This is bad, because all keys must be assumed to be store types:
 
 ```json
 {
diff --git a/doc/manual/src/store/index.md b/doc/manual/src/store/index.md
index 8a5305062..f1e8f1402 100644
--- a/doc/manual/src/store/index.md
+++ b/doc/manual/src/store/index.md
@@ -2,4 +2,4 @@
 
 The *Nix store* is an abstraction to store immutable file system data (such as software packages) that can have dependencies on other such data.
 
-There are multiple implementations of Nix stores with different capabilities, such as the actual filesystem (`/nix/store`) or binary caches.
+There are [multiple types of Nix stores](./types/index.md) with different capabilities, such as the default one on the [local filesystem](./types/local-store.md) (`/nix/store`) or [binary caches](./types/http-binary-cache-store.md).
diff --git a/doc/manual/src/store/types/index.md.in b/doc/manual/src/store/types/index.md.in
index bb166a1fc..b4db553a2 100644
--- a/doc/manual/src/store/types/index.md.in
+++ b/doc/manual/src/store/types/index.md.in
@@ -1,4 +1,6 @@
-Nix supports different types of stores. These are described below.
+Nix supports different types of stores:
+
+@store-types@
 
 ## Store URL format
 
@@ -39,4 +41,3 @@ store as follows:
 
 * Otherwise, use the [local store](#local-store) `/nix/store`.
 
-@stores@
diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix
index 849832b2c..19ff49b64 100644
--- a/doc/manual/utils.nix
+++ b/doc/manual/utils.nix
@@ -1,5 +1,11 @@
 with builtins;
 
+let
+  lowerChars = stringToCharacters "abcdefghijklmnopqrstuvwxyz";
+  upperChars = stringToCharacters "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+  stringToCharacters = s: genList (p: substring p 1 s) (stringLength s);
+in
+
 rec {
   splitLines = s: filter (x: !isList x) (split "\n" s);
 
@@ -18,6 +24,8 @@ rec {
     in
       if replaced == string then string else replaceStringsRec from to replaced;
 
+  toLower = replaceStrings upperChars lowerChars;
+
   squash = replaceStringsRec "\n\n\n" "\n\n";
 
   trim = string:
diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc
index 401acc38e..193972272 100644
--- a/src/libcmd/common-eval-args.cc
+++ b/src/libcmd/common-eval-args.cc
@@ -141,7 +141,7 @@ MixEvalArgs::MixEvalArgs()
         .longName = "eval-store",
         .description =
           R"(
-            The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
+            The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format)
             to use for evaluation, i.e. to store derivations (`.drv` files) and inputs referenced by them.
           )",
         .category = category,
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index ebf2549e4..4ecdda55c 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -4460,7 +4460,7 @@ void EvalState::createBaseEnv()
         .doc = R"(
           Logical file system location of the [Nix store](@docroot@/glossary.md#gloss-store) currently in use.
 
-          This value is determined by the `store` parameter in [Store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md):
+          This value is determined by the `store` parameter in [Store URLs](@docroot@/store/types/index.md#store-url-format):
 
           ```shell-session
           $ nix-instantiate --store 'dummy://?store=/blah' --eval --expr builtins.storeDir
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 838d2aba2..38b0d516c 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -117,10 +117,11 @@ public:
 
     Setting<std::string> storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store",
         R"(
-          The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
+          The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format)
           to use for most operations.
-          See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md)
-          for supported store types and settings.
+          See the
+          [Store Types](@docroot@/store/types/index.md)
+          section of the manual for supported store types and settings.
         )"};
 
     Setting<bool> keepFailed{this, false, "keep-failed",
@@ -759,7 +760,7 @@ public:
         Strings{"https://cache.nixos.org/"},
         "substituters",
         R"(
-          A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) to be used as substituters, separated by whitespace.
+          A list of [URLs of Nix stores](@docroot@/store/types/index.md#store-url-format) to be used as substituters, separated by whitespace.
           A substituter is an additional [store]{@docroot@/glossary.md##gloss-store} from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them.
 
           Substituters are tried based on their priority value, which each substituter can set independently.
@@ -778,7 +779,7 @@ public:
     Setting<StringSet> trustedSubstituters{
         this, {}, "trusted-substituters",
         R"(
-          A list of [Nix store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace.
+          A list of [Nix store URLs](@docroot@/store/types/index.md#store-url-format), separated by whitespace.
           These are not used by default, but users of the Nix daemon can enable them by specifying [`substituters`](#conf-substituters).
 
           Unprivileged users (those set in only [`allowed-users`](#conf-allowed-users) but not [`trusted-users`](#conf-trusted-users)) can pass as `substituters` only those URLs listed in `trusted-substituters`.
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index f16949f42..3d3919882 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -20,7 +20,7 @@
 
 namespace nix {
 
-/* TODO: Separate these store impls into different files, give them better names */
+/* TODO: Separate these store types into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params)
     : RemoteStoreConfig(params)
     , Store(params)
diff --git a/src/libstore/ssh-store-config.hh b/src/libstore/ssh-store-config.hh
index bf55d20cf..4ce4ffc4c 100644
--- a/src/libstore/ssh-store-config.hh
+++ b/src/libstore/ssh-store-config.hh
@@ -20,7 +20,7 @@ struct CommonSSHStoreConfig : virtual StoreConfig
 
     const Setting<std::string> remoteStore{this, "", "remote-store",
         R"(
-          [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
+          [Store URL](@docroot@/store/types/index.md#store-url-format)
           to be used on the remote machine. The default is `auto`
           (i.e. use the Nix daemon or `/nix/store` directly).
         )"};
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 8b6bf9aed..e28baf34e 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -30,7 +30,7 @@
 namespace nix {
 
 /**
- * About the class hierarchy of the store implementations:
+ * About the class hierarchy of the store types:
  *
  * Each store type `Foo` consists of two classes:
  *
@@ -962,7 +962,7 @@ OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * ev
  * - ‘ssh://[user@]<host>’: A remote Nix store accessed by running
  *   ‘nix-store --serve’ via SSH.
  *
- * You can pass parameters to the store implementation by appending
+ * You can pass parameters to the store type by appending
  * ‘?key=value&key=value&...’ to the URI.
  */
 ref<Store> openStore(const std::string & uri = settings.storeUri.get(),
diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index 2418e3f4c..e4bdb8cb3 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -252,7 +252,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
         .tag = Xp::ReadOnlyLocalStore,
         .name = "read-only-local-store",
         .description = R"(
-            Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs.
+            Allow the use of the `read-only` parameter in [local store](@docroot@/store/types/local-store.md) URIs.
         )",
     },
     {
diff --git a/src/nix/nix.md b/src/nix/nix.md
index eb150f03b..749456014 100644
--- a/src/nix/nix.md
+++ b/src/nix/nix.md
@@ -235,10 +235,14 @@ operate are determined as follows:
 
 # Nix stores
 
-Most `nix` subcommands operate on a *Nix store*. These are documented
-in [`nix help-stores`](./nix3-help-stores.md).
+Most `nix` subcommands operate on a *Nix store*.
+The various store types are documented in the
+[Store Types](@docroot@/store/types/index.md)
+section of the manual.
 
-# Shebang interpreter 
+The same information is also available from the [`nix help-stores`](./nix3-help-stores.md) command.
+
+# Shebang interpreter
 
 The `nix` command can be used as a `#!` interpreter.
 Arguments to Nix can be passed on subsequent lines in the script.

From 333ea684b065318aa49aec367c995b3d8c5d65ed Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <silvan.mosberger@tweag.io>
Date: Fri, 1 Dec 2023 01:39:52 +0100
Subject: [PATCH 123/421] Add boost::regex regression test

---
 src/libexpr/tests/primops.cc | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/libexpr/tests/primops.cc b/src/libexpr/tests/primops.cc
index d820b860e..7485fa0d0 100644
--- a/src/libexpr/tests/primops.cc
+++ b/src/libexpr/tests/primops.cc
@@ -814,6 +814,14 @@ namespace nix {
         ASSERT_THAT(*v.listElems()[0], IsStringEq("FOO"));
     }
 
+    TEST_F(PrimOpTest, match5) {
+        // The regex "\\{}" is valid and matches the string "{}".
+        // Caused a regression before when trying to switch from std::regex to boost::regex.
+        // See https://github.com/NixOS/nix/pull/7762#issuecomment-1834303659
+        auto v = eval("builtins.match \"\\\\{}\" \"{}\"");
+        ASSERT_THAT(v, IsListOfSize(0));
+    }
+
     TEST_F(PrimOpTest, attrNames) {
         auto v = eval("builtins.attrNames { x = 1; y = 2; z = 3; a = 2; }");
         ASSERT_THAT(v, IsListOfSize(4));

From d5e934fb73496a2509755be5945a8bcf1730d59d Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Fri, 1 Dec 2023 01:54:48 +0100
Subject: [PATCH 124/421] add redirect to new store page

---
 doc/manual/_redirects | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/_redirects b/doc/manual/_redirects
index 2038671d7..62c693c97 100644
--- a/doc/manual/_redirects
+++ b/doc/manual/_redirects
@@ -31,9 +31,9 @@
 /installation/installation /installation 301!
 
 /package-management/basic-package-mgmt /command-ref/nix-env 301!
-/package-management/channels* /command-ref/nix-channel 301!
+/package-management/channels /command-ref/nix-channel 301!
 /package-management/package-management /package-management 301!
-/package-management/s3-substituter* /command-ref/new-cli/nix3-help-stores#s3-binary-cache-store 301!
+/package-management/s3-substituter /store/types/s3-binary-cache-store 301!
 
 /protocols/protocols /protocols 301!
 

From eff9b12bc296213c3ba824e90869bcafc4103e1c Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Fri, 1 Dec 2023 11:25:22 +0000
Subject: [PATCH 125/421] Further changes

---
 binary-tarball.nix |   6 ++-
 coverage.nix       |  35 ++++++++++++++
 flake.nix          | 115 ++++++++++-----------------------------------
 package.nix        |  88 +++++++++++++++++-----------------
 4 files changed, 108 insertions(+), 136 deletions(-)
 create mode 100644 coverage.nix

diff --git a/binary-tarball.nix b/binary-tarball.nix
index 1fa185519..0053abbca 100644
--- a/binary-tarball.nix
+++ b/binary-tarball.nix
@@ -1,8 +1,8 @@
 { runCommand
-, version
 , system
-, nix
+, buildPackages
 , cacert
+, nix
 }:
 
 let
@@ -11,6 +11,8 @@ let
     rootPaths = [ nix cacert ];
   };
 
+  inherit (nix) version;
+
   env = {
     meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
   };
diff --git a/coverage.nix b/coverage.nix
new file mode 100644
index 000000000..2390ef52d
--- /dev/null
+++ b/coverage.nix
@@ -0,0 +1,35 @@
+{ lib
+, releaseTools
+, nix
+, stdenv
+}:
+
+let
+  inherit (nix) version;
+
+in
+
+releaseTools.coverageAnalysis {
+  name = "nix-coverage-${version}";
+
+  inherit (nix)
+    src
+    configureFlags
+    nativeBuildInputs
+    buildInputs
+    #checkInputs
+    ;
+
+  enableParallelBuilding = true;
+
+  dontInstall = false;
+
+  doInstallCheck = true;
+  installCheckTarget = "installcheck"; # work around buggy detection in stdenv
+
+  lcovFilter = [ "*/boost/*" "*-tab.*" ];
+
+  hardeningDisable = ["fortify"];
+
+  NIX_CFLAGS_COMPILE = "-DCOVERAGE=1";
+}
diff --git a/flake.nix b/flake.nix
index 544a07ba6..c0841a76d 100644
--- a/flake.nix
+++ b/flake.nix
@@ -479,60 +479,25 @@
         dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage);
 
         # Line coverage analysis.
-        coverage =
-          with nixpkgsFor.x86_64-linux.native;
-          with commonDeps { inherit pkgs; };
-
-          releaseTools.coverageAnalysis {
-            name = "nix-coverage-${version}";
-
-            src = nixSrc;
-
-            configureFlags = testConfigureFlags;
-
-            enableParallelBuilding = true;
-
-            nativeBuildInputs = nativeBuildDeps;
-            buildInputs = buildDeps ++ propagatedDeps ++ awsDeps ++ checkDeps;
-
-            dontInstall = false;
-
-            doInstallCheck = true;
-            installCheckTarget = "installcheck"; # work around buggy detection in stdenv
-
-            lcovFilter = [ "*/boost/*" "*-tab.*" ];
-
-            hardeningDisable = ["fortify"];
-
-            NIX_CFLAGS_COMPILE = "-DCOVERAGE=1";
-          };
+        coverage = nixpkgsFor.x86_64-linux.native.callPackage ./coverage.nix {};
 
         # API docs for Nix's unstable internal C++ interfaces.
-        internal-api-docs =
-          with nixpkgsFor.x86_64-linux.native;
-          with commonDeps { inherit pkgs; };
+        internal-api-docs = nixpkgsFor.x86_64-linux.native.nix.overrideAttrs (old: {
+          pname = "nix-internal-api-docs";
 
-          stdenv.mkDerivation {
-            pname = "nix-internal-api-docs";
-            inherit version;
+          configureFlags = old.configureFlags ++ [ "--enable-internal-api-docs" ];
+          nativeBuildInputs = old.nativeBuildInputs ++ [ nixpkgsFor.x86_64-linux.native.doxygen ];
 
-            src = nixSrc;
+          dontBuild = true;
+          doCheck = false;
 
-            configureFlags = testConfigureFlags ++ internalApiDocsConfigureFlags;
+          installTargets = [ "internal-api-html" ];
 
-            nativeBuildInputs = nativeBuildDeps;
-            buildInputs = buildDeps ++ propagatedDeps
-              ++ awsDeps ++ checkDeps ++ internalApiDocsDeps;
-
-            dontBuild = true;
-
-            installTargets = [ "internal-api-html" ];
-
-            postInstall = ''
-              mkdir -p $out/nix-support
-              echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products
-            '';
-          };
+          postInstall = ''
+            mkdir -p $out/nix-support
+            echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products
+          '';
+        });
 
         # System tests.
         tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // {
@@ -540,7 +505,9 @@
           # Make sure that nix-env still produces the exact same result
           # on a particular version of Nixpkgs.
           evalNixpkgs =
-            with nixpkgsFor.x86_64-linux.native;
+            let
+              inherit (nixpkgsFor.x86_64-linux.native) runCommand nix nixpkgs-regression;
+            in
             runCommand "eval-nixos" { buildInputs = [ nix ]; }
               ''
                 type -p nix-env
@@ -627,47 +594,17 @@
           stdenvs)));
 
       devShells = let
-        makeShell = pkgs: stdenv:
-          let
-            canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
-          in
-          with commonDeps { inherit pkgs; };
-          stdenv.mkDerivation {
-            name = "nix";
+        makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; }).overrideAttrs (_: {
+          installFlags = "sysconfdir=$(out)/etc";
+          shellHook = ''
+            PATH=$prefix/bin:$PATH
+            unset PYTHONPATH
+            export MANPATH=$out/share/man:$MANPATH
 
-            outputs = [ "out" "dev" "doc" ]
-              ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "check";
-
-            nativeBuildInputs = nativeBuildDeps
-              ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear
-              ++ lib.optional
-                (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform)
-                pkgs.buildPackages.clang-tools
-              # We want changelog-d in the shell even if the current build doesn't need it
-              ++ lib.optional (officialRelease || ! buildUnreleasedNotes) changelog-d
-              ;
-
-            buildInputs = buildDeps ++ propagatedDeps
-              ++ awsDeps ++ checkDeps ++ internalApiDocsDeps;
-
-            configureFlags = configureFlags
-              ++ testConfigureFlags ++ internalApiDocsConfigureFlags
-              ++ lib.optional (!canRunInstalled) "--disable-doc-gen";
-
-            enableParallelBuilding = true;
-
-            installFlags = "sysconfdir=$(out)/etc";
-
-            shellHook =
-              ''
-                PATH=$prefix/bin:$PATH
-                unset PYTHONPATH
-                export MANPATH=$out/share/man:$MANPATH
-
-                # Make bash completion work.
-                XDG_DATA_DIRS+=:$out/share
-              '';
-          };
+            # Make bash completion work.
+            XDG_DATA_DIRS+=:$out/share
+          '';
+        });
         in
         forAllSystems (system:
           let
diff --git a/package.nix b/package.nix
index 8d62120fb..bed77ba3b 100644
--- a/package.nix
+++ b/package.nix
@@ -41,16 +41,12 @@
 }:
 
 let
-
   version = lib.fileContents ./.version + versionSuffix;
-
-  inherit (stdenv.hostPlatform) isStatic;
-
   canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
 in
 
 stdenv.mkDerivation (finalAttrs: {
-  name = "nix-${version}";
+  pname = "nix";
   inherit version;
 
   src =
@@ -103,17 +99,12 @@ stdenv.mkDerivation (finalAttrs: {
     bison
     flex
     (lib.getBin lowdown)
+    jq # Also for custom mdBook preprocessor.
     mdbook
     mdbook-linkcheck
     autoconf-archive
     autoreconfHook
     pkg-config
-
-    # Tests
-    git
-    mercurial # FIXME: remove? only needed for tests
-    jq # Also for custom mdBook preprocessor.
-    openssh # only needed for tests (ssh-keygen)
   ]
   ++ lib.optional stdenv.hostPlatform.isLinux util-linux
   # Official releases don't have rl-next, so we don't need to compile a changelog
@@ -133,19 +124,29 @@ stdenv.mkDerivation (finalAttrs: {
     sqlite
     xz
   ]
-  ++ lib.optionals stdenv.isLinux [libseccomp]
+  ++ lib.optional stdenv.isLinux libseccomp
   ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid
   # There have been issues building these dependencies
-  ++ lib.optionals (stdenv.hostPlatform == stdenv.buildPlatform) (lib.optional (stdenv.isLinux || stdenv.isDarwin)
+  ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin))
     (aws-sdk-cpp.override {
       apis = ["s3" "transfer"];
       customMemoryManagement = false;
-    }))
-  ++ lib.optionals finalAttrs.doCheck [
+    })
+  ;
+
+  doCheck = true;
+
+  checkInputs = [
     gtest
     rapidcheck
   ];
 
+  nativeCheckInputs = [
+    git
+    mercurial # FIXME: remove? only needed for tests
+    openssh # only needed for tests (ssh-keygen)
+  ];
+
   propagatedBuildInputs = [
     boehmgc
     nlohmann_json
@@ -153,52 +154,49 @@ stdenv.mkDerivation (finalAttrs: {
 
   disallowedReferences = [ boost ];
 
-  preConfigure = lib.optionalString (! stdenv.hostPlatform.isStatic)
-    ''
-      # Copy libboost_context so we don't get all of Boost in our closure.
-      # https://github.com/NixOS/nixpkgs/issues/45462
-      mkdir -p $out/lib
-      cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib
-      rm -f $out/lib/*.a
-      ${lib.optionalString stdenv.hostPlatform.isLinux ''
-        chmod u+w $out/lib/*.so.*
-        patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
-      ''}
-      ${lib.optionalString stdenv.hostPlatform.isDarwin ''
-        for LIB in $out/lib/*.dylib; do
-          chmod u+w $LIB
-          install_name_tool -id $LIB $LIB
-          install_name_tool -delete_rpath ${boost}/lib/ $LIB || true
-        done
-        install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
-      ''}
-    '';
+  preConfigure = lib.optionalString (! stdenv.hostPlatform.isStatic) ''
+    # Copy libboost_context so we don't get all of Boost in our closure.
+    # https://github.com/NixOS/nixpkgs/issues/45462
+    mkdir -p $out/lib
+    cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib
+    rm -f $out/lib/*.a
+    ${lib.optionalString stdenv.hostPlatform.isLinux ''
+      chmod u+w $out/lib/*.so.*
+      patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
+    ''}
+    ${lib.optionalString stdenv.hostPlatform.isDarwin ''
+      for LIB in $out/lib/*.dylib; do
+        chmod u+w $LIB
+        install_name_tool -id $LIB $LIB
+        install_name_tool -delete_rpath ${boost}/lib/ $LIB || true
+      done
+      install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
+    ''}
+  '';
 
   configureFlags =
     lib.optionals stdenv.isLinux [
       "--with-boost=${boost}/lib"
       "--with-sandbox-shell=${sh}/bin/busybox"
     ]
-    ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [
+    ++ lib.optional (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux"))
       "LDFLAGS=-fuse-ld=gold"
-    ]
     ++ [ "--sysconfdir=/etc" ]
     ++ lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell"
     ++ [ (lib.enableFeature finalAttrs.doCheck "tests") ]
-    ++ lib.optionals finalAttrs.doCheck ([ "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" ]
-    ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
-      "--enable-install-unit-tests"
-      "--with-check-bin-dir=${builtins.placeholder "check"}/bin"
-      "--with-check-lib-dir=${builtins.placeholder "check"}/lib"
-    ])
+    ++ lib.optionals finalAttrs.doCheck (
+      [ "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" ]
+      ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
+        "--enable-install-unit-tests"
+        "--with-check-bin-dir=${builtins.placeholder "check"}/bin"
+        "--with-check-lib-dir=${builtins.placeholder "check"}/lib"
+      ])
     ++ lib.optional (!canRunInstalled) "--disable-doc-gen";
 
   enableParallelBuilding = true;
 
   makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1";
 
-  doCheck = true;
-
   installFlags = "sysconfdir=$(out)/etc";
 
   postInstall = ''

From ea2dd166235e049699cf7f70c243c2b83089f824 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Fri, 1 Dec 2023 15:35:21 +0100
Subject: [PATCH 126/421] Use a proper enum rather than a boolean in
 runProgramInStore

Makes the call-site much easier to understand.
---
 src/nix/develop.cc | 2 +-
 src/nix/fmt.cc     | 2 +-
 src/nix/run.cc     | 8 ++++----
 src/nix/run.hh     | 7 ++++++-
 4 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index ae9be79a3..606b044b0 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -662,7 +662,7 @@ struct CmdDevelop : Common, MixEnvironment
             }
         }
 
-        runProgramInStore(store, true, shell, args, buildEnvironment.getSystem());
+        runProgramInStore(store, UseSearchPath::Use, shell, args, buildEnvironment.getSystem());
     }
 };
 
diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc
index 396c93dbb..059904150 100644
--- a/src/nix/fmt.cc
+++ b/src/nix/fmt.cc
@@ -49,7 +49,7 @@ struct CmdFmt : SourceExprCommand {
             }
         }
 
-        runProgramInStore(store, false, app.program, programArgs);
+        runProgramInStore(store, UseSearchPath::DontUse, app.program, programArgs);
     };
 };
 
diff --git a/src/nix/run.cc b/src/nix/run.cc
index d531f712d..efc0c56a1 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -25,7 +25,7 @@ std::string chrootHelperName = "__run_in_chroot";
 namespace nix {
 
 void runProgramInStore(ref<Store> store,
-    bool search,
+    UseSearchPath useSearchPath,
     const std::string & program,
     const Strings & args,
     std::optional<std::string_view> system)
@@ -59,7 +59,7 @@ void runProgramInStore(ref<Store> store,
     if (system)
         setPersonality(*system);
 
-    if (search)
+    if (useSearchPath == UseSearchPath::Use)
         execvp(program.c_str(), stringsToCharPtrs(args).data());
     else
         execv(program.c_str(), stringsToCharPtrs(args).data());
@@ -136,7 +136,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment
         Strings args;
         for (auto & arg : command) args.push_back(arg);
 
-        runProgramInStore(store, true, *command.begin(), args);
+        runProgramInStore(store, UseSearchPath::Use, *command.begin(), args);
     }
 };
 
@@ -198,7 +198,7 @@ struct CmdRun : InstallableValueCommand
         Strings allArgs{app.program};
         for (auto & i : args) allArgs.push_back(i);
 
-        runProgramInStore(store, false, app.program, allArgs);
+        runProgramInStore(store, UseSearchPath::DontUse, app.program, allArgs);
     }
 };
 
diff --git a/src/nix/run.hh b/src/nix/run.hh
index c62287e7e..a55917b06 100644
--- a/src/nix/run.hh
+++ b/src/nix/run.hh
@@ -5,8 +5,13 @@
 
 namespace nix {
 
+enum struct UseSearchPath {
+    Use,
+    DontUse
+};
+
 void runProgramInStore(ref<Store> store,
-    bool search,
+    UseSearchPath useSearchPath,
     const std::string & program,
     const Strings & args,
     std::optional<std::string_view> system = std::nullopt);

From d59bdbe4fd757d99b6625db1d3560a39a371d9e9 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 1 Dec 2023 10:20:19 -0500
Subject: [PATCH 127/421] Add two missing `#include "nar-info.hh"`

GitHub's racy CI caused this oversight to sneak through.
---
 src/libstore/tests/nar-info.cc | 1 +
 src/nix/path-info.cc           | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/libstore/tests/nar-info.cc b/src/libstore/tests/nar-info.cc
index c5b21d56b..88e6e1add 100644
--- a/src/libstore/tests/nar-info.cc
+++ b/src/libstore/tests/nar-info.cc
@@ -2,6 +2,7 @@
 #include <gtest/gtest.h>
 
 #include "path-info.hh"
+#include "nar-info.hh"
 
 #include "tests/characterization.hh"
 #include "tests/libstore.hh"
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index 080d6bbf1..5f10cfb61 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -2,6 +2,7 @@
 #include "shared.hh"
 #include "store-api.hh"
 #include "common-args.hh"
+#include "nar-info.hh"
 
 #include <algorithm>
 #include <array>

From 91b6833686a6a6d9eac7f3f66393ec89ef1d3b57 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 25 Aug 2023 10:20:28 -0400
Subject: [PATCH 128/421] Move tests to separate directories, and document

Today, with the tests inside a `tests` intermingled with the
corresponding library's source code, we have a few problems:

- We have to be careful that wildcards don't end up with tests being
  built as part of Nix proper, or test headers being installed as part
  of Nix proper.

- Tests in libraries but not executables is not right:

  - It means each executable runs the previous unit tests again, because
    it needs the libraries.

  - It doesn't work right on Windows, which doesn't want you to load a
    DLL just for the side global variable . It could be made to work
    with the dlopen equivalent, but that's gross!

This reorg solves these problems.

There is a remaining problem which is that sibbling headers (like
`hash.hh` the test header vs `hash.hh` the main `libnixutil` header) end
up shadowing each other. This PR doesn't solve that. That is left as
future work for a future PR.

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 .gitignore                                    |   6 +-
 Makefile                                      |  10 ++-
 doc/internal-api/doxygen.cfg.in               |  12 ++-
 doc/manual/src/contributing/testing.md        |  67 ++++++++++----
 flake.nix                                     |   2 +-
 mk/common-test.sh                             |   2 +-
 mk/programs.mk                                |   2 +-
 src/libexpr/tests/local.mk                    |  23 -----
 src/libstore/tests/local.mk                   |  37 --------
 src/libutil/tests/local.mk                    |  41 ---------
 tests/unit/libexpr-support/local.mk           |  23 +++++
 .../unit/libexpr-support}/tests/libexpr.hh    |   0
 .../libexpr-support/tests/value/context.cc    |  30 +++++++
 .../libexpr-support}/tests/value/context.hh   |   2 +-
 .../unit/libexpr}/derived-path.cc             |   0
 .../unit/libexpr}/error_traces.cc             |   0
 .../tests => tests/unit/libexpr}/flakeref.cc  |   0
 .../tests => tests/unit/libexpr}/json.cc      |   0
 tests/unit/libexpr/local.mk                   |  36 ++++++++
 .../tests => tests/unit/libexpr}/primops.cc   |   0
 .../unit/libexpr}/search-path.cc              |   0
 .../tests => tests/unit/libexpr}/trivial.cc   |   0
 .../unit/libexpr}/value/context.cc            |  30 -------
 .../unit/libexpr}/value/print.cc              |   0
 tests/unit/libstore-support/local.mk          |  21 +++++
 .../libstore-support/tests/derived-path.cc    |  57 ++++++++++++
 .../libstore-support}/tests/derived-path.hh   |   0
 .../unit/libstore-support}/tests/libstore.hh  |   0
 .../libstore-support/tests/outputs-spec.cc    |  24 +++++
 .../libstore-support}/tests/outputs-spec.hh   |   2 +-
 tests/unit/libstore-support/tests/path.cc     |  82 ++++++++++++++++++
 .../unit/libstore-support}/tests/path.hh      |   3 +
 .../unit/libstore-support}/tests/protocol.hh  |   2 +-
 .../unit/libstore}/common-protocol.cc         |   0
 .../data}/common-protocol/content-address.bin | Bin
 .../data}/common-protocol/drv-output.bin      | Bin
 .../optional-content-address.bin              | Bin
 .../common-protocol/optional-store-path.bin   | Bin
 .../data}/common-protocol/realisation.bin     | Bin
 .../libstore/data}/common-protocol/set.bin    | Bin
 .../data}/common-protocol/store-path.bin      | Bin
 .../libstore/data}/common-protocol/string.bin | Bin
 .../libstore/data}/common-protocol/vector.bin | Bin
 .../derivation/bad-old-version-dyn-deps.drv   |   0
 .../libstore/data}/derivation/bad-version.drv |   0
 .../data}/derivation/dynDerivationDeps.drv    |   0
 .../data}/derivation/dynDerivationDeps.json   |   0
 .../data}/derivation/output-caFixedFlat.json  |   0
 .../data}/derivation/output-caFixedNAR.json   |   0
 .../data}/derivation/output-caFixedText.json  |   0
 .../data}/derivation/output-caFloating.json   |   0
 .../data}/derivation/output-deferred.json     |   0
 .../data}/derivation/output-impure.json       |   0
 .../derivation/output-inputAddressed.json     |   0
 .../unit/libstore/data}/derivation/simple.drv |   0
 .../libstore/data}/derivation/simple.json     |   0
 .../unit/libstore/data}/nar-info/impure.json  |   0
 .../unit/libstore/data}/nar-info/pure.json    |   0
 .../unit/libstore/data}/path-info/impure.json |   0
 .../unit/libstore/data}/path-info/pure.json   |   0
 .../data}/serve-protocol/build-result-2.2.bin | Bin
 .../data}/serve-protocol/build-result-2.3.bin | Bin
 .../data}/serve-protocol/build-result-2.6.bin | Bin
 .../data}/serve-protocol/content-address.bin  | Bin
 .../data}/serve-protocol/drv-output.bin       | Bin
 .../optional-content-address.bin              | Bin
 .../serve-protocol/optional-store-path.bin    | Bin
 .../data}/serve-protocol/realisation.bin      | Bin
 .../libstore/data}/serve-protocol/set.bin     | Bin
 .../data}/serve-protocol/store-path.bin       | Bin
 .../libstore/data}/serve-protocol/string.bin  | Bin
 .../libstore/data}/serve-protocol/vector.bin  | Bin
 .../worker-protocol/build-result-1.27.bin     | Bin
 .../worker-protocol/build-result-1.28.bin     | Bin
 .../worker-protocol/build-result-1.29.bin     | Bin
 .../data}/worker-protocol/content-address.bin | Bin
 .../worker-protocol/derived-path-1.29.bin     | Bin
 .../worker-protocol/derived-path-1.30.bin     | Bin
 .../data}/worker-protocol/drv-output.bin      | Bin
 .../keyed-build-result-1.29.bin               | Bin
 .../optional-content-address.bin              | Bin
 .../worker-protocol/optional-store-path.bin   | Bin
 .../worker-protocol/optional-trusted-flag.bin | Bin
 .../data}/worker-protocol/realisation.bin     | Bin
 .../libstore/data}/worker-protocol/set.bin    | Bin
 .../data}/worker-protocol/store-path.bin      | Bin
 .../libstore/data}/worker-protocol/string.bin | Bin
 .../unkeyed-valid-path-info-1.15.bin          | Bin
 .../worker-protocol/valid-path-info-1.15.bin  | Bin
 .../worker-protocol/valid-path-info-1.16.bin  | Bin
 .../libstore/data}/worker-protocol/vector.bin | Bin
 .../unit/libstore}/derivation.cc              |   2 +-
 .../unit/libstore}/derived-path.cc            |  53 -----------
 .../unit/libstore}/downstream-placeholder.cc  |   0
 tests/unit/libstore/local.mk                  |  31 +++++++
 .../tests => tests/unit/libstore}/machines.cc |   4 +-
 .../unit/libstore}/nar-info-disk-cache.cc     |   0
 .../tests => tests/unit/libstore}/nar-info.cc |   2 +-
 .../unit/libstore}/outputs-spec.cc            |  27 +-----
 .../unit/libstore}/path-info.cc               |   2 +-
 .../tests => tests/unit/libstore}/path.cc     |  73 ----------------
 .../unit/libstore}/references.cc              |   0
 .../unit/libstore}/serve-protocol.cc          |   0
 .../libstore}/test-data/machines.bad_format   |   0
 .../unit/libstore}/test-data/machines.valid   |   0
 .../unit/libstore}/worker-protocol.cc         |   0
 tests/unit/libutil-support/local.mk           |  19 ++++
 .../tests/characterization.hh                 |   4 +-
 tests/unit/libutil-support/tests/hash.cc      |  20 +++++
 .../unit/libutil-support}/tests/hash.hh       |   0
 .../tests => tests/unit/libutil}/args.cc      |   6 +-
 .../unit/libutil}/canon-path.cc               |   0
 .../unit/libutil}/chunked-vector.cc           |   0
 .../tests => tests/unit/libutil}/closure.cc   |   0
 .../unit/libutil}/compression.cc              |   0
 .../tests => tests/unit/libutil}/config.cc    |   0
 .../unit/libutil/data}/git/check-data.sh      |   2 +-
 .../libutil/data}/git/hello-world-blob.bin    | Bin
 .../unit/libutil/data}/git/hello-world.bin    | Bin
 .../unit/libutil/data}/git/tree.bin           | Bin
 .../unit/libutil/data}/git/tree.txt           |   0
 .../tests => tests/unit/libutil}/git.cc       |   6 +-
 .../tests => tests/unit/libutil}/hash.cc      |  20 +----
 .../tests => tests/unit/libutil}/hilite.cc    |   0
 .../unit/libutil}/json-utils.cc               |   0
 tests/unit/libutil/local.mk                   |  31 +++++++
 .../tests => tests/unit/libutil}/logging.cc   |   0
 .../tests => tests/unit/libutil}/lru-cache.cc |   0
 .../tests => tests/unit/libutil}/pool.cc      |   0
 .../unit/libutil}/references.cc               |   0
 .../unit/libutil}/suggestions.cc              |   0
 .../tests => tests/unit/libutil}/tests.cc     |   0
 .../tests => tests/unit/libutil}/url.cc       |   0
 .../unit/libutil}/xml-writer.cc               |   0
 134 files changed, 464 insertions(+), 352 deletions(-)
 delete mode 100644 src/libexpr/tests/local.mk
 delete mode 100644 src/libstore/tests/local.mk
 delete mode 100644 src/libutil/tests/local.mk
 create mode 100644 tests/unit/libexpr-support/local.mk
 rename {src/libexpr => tests/unit/libexpr-support}/tests/libexpr.hh (100%)
 create mode 100644 tests/unit/libexpr-support/tests/value/context.cc
 rename {src/libexpr => tests/unit/libexpr-support}/tests/value/context.hh (95%)
 rename {src/libexpr/tests => tests/unit/libexpr}/derived-path.cc (100%)
 rename {src/libexpr/tests => tests/unit/libexpr}/error_traces.cc (100%)
 rename {src/libexpr/tests => tests/unit/libexpr}/flakeref.cc (100%)
 rename {src/libexpr/tests => tests/unit/libexpr}/json.cc (100%)
 create mode 100644 tests/unit/libexpr/local.mk
 rename {src/libexpr/tests => tests/unit/libexpr}/primops.cc (100%)
 rename {src/libexpr/tests => tests/unit/libexpr}/search-path.cc (100%)
 rename {src/libexpr/tests => tests/unit/libexpr}/trivial.cc (100%)
 rename {src/libexpr/tests => tests/unit/libexpr}/value/context.cc (83%)
 rename {src/libexpr/tests => tests/unit/libexpr}/value/print.cc (100%)
 create mode 100644 tests/unit/libstore-support/local.mk
 create mode 100644 tests/unit/libstore-support/tests/derived-path.cc
 rename {src/libstore => tests/unit/libstore-support}/tests/derived-path.hh (100%)
 rename {src/libstore => tests/unit/libstore-support}/tests/libstore.hh (100%)
 create mode 100644 tests/unit/libstore-support/tests/outputs-spec.cc
 rename {src/libstore => tests/unit/libstore-support}/tests/outputs-spec.hh (89%)
 create mode 100644 tests/unit/libstore-support/tests/path.cc
 rename {src/libstore => tests/unit/libstore-support}/tests/path.hh (82%)
 rename {src/libstore => tests/unit/libstore-support}/tests/protocol.hh (96%)
 rename {src/libstore/tests => tests/unit/libstore}/common-protocol.cc (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/content-address.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/drv-output.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/optional-content-address.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/optional-store-path.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/realisation.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/set.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/store-path.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/string.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/vector.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/bad-old-version-dyn-deps.drv (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/bad-version.drv (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/dynDerivationDeps.drv (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/dynDerivationDeps.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFixedFlat.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFixedNAR.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFixedText.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFloating.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-deferred.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-impure.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-inputAddressed.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/simple.drv (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/simple.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/nar-info/impure.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/nar-info/pure.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/path-info/impure.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/path-info/pure.json (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/build-result-2.2.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/build-result-2.3.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/build-result-2.6.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/content-address.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/drv-output.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/optional-content-address.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/optional-store-path.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/realisation.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/set.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/store-path.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/string.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/vector.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/build-result-1.27.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/build-result-1.28.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/build-result-1.29.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/content-address.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/derived-path-1.29.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/derived-path-1.30.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/drv-output.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/keyed-build-result-1.29.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/optional-content-address.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/optional-store-path.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/optional-trusted-flag.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/realisation.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/set.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/store-path.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/string.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/unkeyed-valid-path-info-1.15.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/valid-path-info-1.15.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/valid-path-info-1.16.bin (100%)
 rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/vector.bin (100%)
 rename {src/libstore/tests => tests/unit/libstore}/derivation.cc (99%)
 rename {src/libstore/tests => tests/unit/libstore}/derived-path.cc (66%)
 rename {src/libstore/tests => tests/unit/libstore}/downstream-placeholder.cc (100%)
 create mode 100644 tests/unit/libstore/local.mk
 rename {src/libstore/tests => tests/unit/libstore}/machines.cc (97%)
 rename {src/libstore/tests => tests/unit/libstore}/nar-info-disk-cache.cc (100%)
 rename {src/libstore/tests => tests/unit/libstore}/nar-info.cc (98%)
 rename {src/libstore/tests => tests/unit/libstore}/outputs-spec.cc (92%)
 rename {src/libstore/tests => tests/unit/libstore}/path-info.cc (97%)
 rename {src/libstore/tests => tests/unit/libstore}/path.cc (59%)
 rename {src/libstore/tests => tests/unit/libstore}/references.cc (100%)
 rename {src/libstore/tests => tests/unit/libstore}/serve-protocol.cc (100%)
 rename {src/libstore/tests => tests/unit/libstore}/test-data/machines.bad_format (100%)
 rename {src/libstore/tests => tests/unit/libstore}/test-data/machines.valid (100%)
 rename {src/libstore/tests => tests/unit/libstore}/worker-protocol.cc (100%)
 create mode 100644 tests/unit/libutil-support/local.mk
 rename {src/libutil => tests/unit/libutil-support}/tests/characterization.hh (95%)
 create mode 100644 tests/unit/libutil-support/tests/hash.cc
 rename {src/libutil => tests/unit/libutil-support}/tests/hash.hh (100%)
 rename {src/libutil/tests => tests/unit/libutil}/args.cc (98%)
 rename {src/libutil/tests => tests/unit/libutil}/canon-path.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/chunked-vector.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/closure.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/compression.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/config.cc (100%)
 rename {unit-test-data/libutil => tests/unit/libutil/data}/git/check-data.sh (98%)
 rename {unit-test-data/libutil => tests/unit/libutil/data}/git/hello-world-blob.bin (100%)
 rename {unit-test-data/libutil => tests/unit/libutil/data}/git/hello-world.bin (100%)
 rename {unit-test-data/libutil => tests/unit/libutil/data}/git/tree.bin (100%)
 rename {unit-test-data/libutil => tests/unit/libutil/data}/git/tree.txt (100%)
 rename {src/libutil/tests => tests/unit/libutil}/git.cc (97%)
 rename {src/libutil/tests => tests/unit/libutil}/hash.cc (92%)
 rename {src/libutil/tests => tests/unit/libutil}/hilite.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/json-utils.cc (100%)
 create mode 100644 tests/unit/libutil/local.mk
 rename {src/libutil/tests => tests/unit/libutil}/logging.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/lru-cache.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/pool.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/references.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/suggestions.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/tests.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/url.cc (100%)
 rename {src/libutil/tests => tests/unit/libutil}/xml-writer.cc (100%)

diff --git a/.gitignore b/.gitignore
index 767a5d6ed..d9f9d949b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -45,14 +45,14 @@ perl/Makefile.config
 /src/libexpr/parser-tab.hh
 /src/libexpr/parser-tab.output
 /src/libexpr/nix.tbl
-/src/libexpr/tests/libnixexpr-tests
+/tests/unit/libexpr/libnixexpr-tests
 
 # /src/libstore/
 *.gen.*
-/src/libstore/tests/libnixstore-tests
+/tests/unit/libstore/libnixstore-tests
 
 # /src/libutil/
-/src/libutil/tests/libnixutil-tests
+/tests/unit/libutil/libnixutil-tests
 
 /src/nix/nix
 
diff --git a/Makefile b/Makefile
index 92727bea5..eea297c89 100644
--- a/Makefile
+++ b/Makefile
@@ -25,11 +25,13 @@ makefiles = \
 endif
 
 ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes)
-UNIT_TEST_ENV = _NIX_TEST_UNIT_DATA=unit-test-data
 makefiles += \
-  src/libutil/tests/local.mk \
-  src/libstore/tests/local.mk \
-  src/libexpr/tests/local.mk
+  tests/unit/libutil/local.mk \
+  tests/unit/libutil-support/local.mk \
+  tests/unit/libstore/local.mk \
+  tests/unit/libstore-support/local.mk \
+  tests/unit/libexpr/local.mk \
+  tests/unit/libexpr-support/local.mk
 endif
 
 ifeq ($(ENABLE_TESTS), yes)
diff --git a/doc/internal-api/doxygen.cfg.in b/doc/internal-api/doxygen.cfg.in
index 599be2470..ad5af97e6 100644
--- a/doc/internal-api/doxygen.cfg.in
+++ b/doc/internal-api/doxygen.cfg.in
@@ -39,17 +39,21 @@ INPUT                  = \
   src/libcmd \
   src/libexpr \
   src/libexpr/flake \
-  src/libexpr/tests \
-  src/libexpr/tests/value \
+  tests/unit/libexpr \
+  tests/unit/libexpr/value \
+  tests/unit/libexpr/test \
+  tests/unit/libexpr/test/value \
   src/libexpr/value \
   src/libfetchers \
   src/libmain \
   src/libstore \
   src/libstore/build \
   src/libstore/builtins \
-  src/libstore/tests \
+  tests/unit/libstore \
+  tests/unit/libstore/test \
   src/libutil \
-  src/libutil/tests \
+  tests/unit/libutil \
+  tests/unit/libutil/test \
   src/nix \
   src/nix-env \
   src/nix-store
diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md
index 0b45b88a3..d8d162379 100644
--- a/doc/manual/src/contributing/testing.md
+++ b/doc/manual/src/contributing/testing.md
@@ -20,6 +20,7 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks.
 
 [googletest]: https://google.github.io/googletest/
 [rapidcheck]: https://github.com/emil-e/rapidcheck
+[property testing]: https://en.wikipedia.org/wiki/Property_testing
 
 ### Source and header layout
 
@@ -28,34 +29,50 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks.
 > ```
 > src
 > ├── libexpr
+> │   ├── local.mk
 > │   ├── value/context.hh
 > │   ├── value/context.cc
+> │   …
+> │
+> ├── tests
 > │   │
 > │   …
->     └── tests
-> │       ├── value/context.hh
-> │       ├── value/context.cc
+> │   └── unit
+> │       ├── libutil
+> │       │   ├── local.mk
+> │       │   …
+> │       │   └── data
+> │       │       ├── git/tree.txt
+> │       │       …
 > │       │
-> │       …
-> │
-> ├── unit-test-data
-> │   ├── libstore
-> │   │   ├── worker-protocol/content-address.bin
-> │   │   …
-> │   …
+> │       ├── libexpr-support
+> │       │   ├── local.mk
+> │       │   └── tests
+> │       │       ├── value/context.hh
+> │       │       ├── value/context.cc
+> │       │       …
+> │       │
+> │       ├── libexpr
+> │       …   ├── local.mk
+> │           ├── value/context.cc
+> │           …
 > …
 > ```
 
-The unit tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_shortname}/tests` within the directory for the library (`src/${library_shortname}`).
+The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `tests/unit/${library_name_without-nix}`.
+Given a interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `tests/unit/libexpr/tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `tests/unit/libexpr-support/tests/value/context.{hh,cc}`.
 
-The data is in `unit-test-data`, with one subdir per library, with the same name as where the code goes.
-For example, `libnixstore` code is in `src/libstore`, and its test data is in `unit-test-data/libstore`.
-The path to the `unit-test-data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`.
+Data for unit tests is stored in a `data` subdir of the directory for each unit test executable.
+For example, `libnixstore` code is in `src/libstore`, and its test data is in `tests/unit/libstore/data`.
+The path to the `tests/unit/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`.
+Note that each executable only gets the data for its tests.
 
-> **Note**
-> Due to the way googletest works, downstream unit test executables will actually include and re-run upstream library tests.
-> Therefore it is important that the same value for `_NIX_TEST_UNIT_DATA` be used with the tests for each library.
-> That is why we have the test data nested within a single `unit-test-data` directory.
+The unit test libraries are in `tests/unit/${library_name_without-nix}-lib`.
+All headers are in a `tests` subdirectory so they are included with `#include "tests/"`.
+
+The use of all these separate directories for the unit tests might seem inconvenient, as for example the tests are not "right next to" the part of the code they are testing.
+But organizing the tests this way has one big benefit:
+there is no risk of any build-system wildcards for the library accidentally picking up test code that should not built and installed as part of the library.
 
 ### Running tests
 
@@ -69,7 +86,7 @@ See [functional characterisation testing](#characterisation-testing-functional)
 Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used.
 For example:
 ```shell-session
-$ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN
+$ _NIX_TEST_ACCEPT=1 make libstore-tests_RUN
 ...
 [  SKIPPED ] WorkerProtoTest.string_read
 [  SKIPPED ] WorkerProtoTest.string_write
@@ -80,6 +97,18 @@ $ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN
 will regenerate the "golden master" expected result for the `libnixstore` characterisation tests.
 The characterisation tests will mark themselves "skipped" since they regenerated the expected result instead of actually testing anything.
 
+### Unit test support libraries
+
+There are headers and code which are not just used to test the library in question, but also downstream libraries.
+For example, we do [property testing] with the [rapidcheck] library.
+This requires writing `Arbitrary` "instances", which are used to describe how to generate values of a given type for the sake of running property tests.
+Because types contain other types, `Arbitrary` "instances" for some type are not just useful for testing that type, but also any other type that contains it.
+Downstream types frequently contain upstream types, so it is very important that we share arbitrary instances so that downstream libraries' property tests can also use them.
+
+It is important that these testing libraries don't contain any actual tests themselves.
+On some platforms they would be run as part of every test executable that uses them, which is redundant.
+On other platforms they wouldn't be run at all.
+
 ## Functional tests
 
 The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`.
diff --git a/flake.nix b/flake.nix
index 822b3d31e..e2e510cbc 100644
--- a/flake.nix
+++ b/flake.nix
@@ -93,7 +93,7 @@
           ./misc
           ./precompiled-headers.h
           ./src
-          ./unit-test-data
+          ./tests/unit
           ./COPYING
           ./scripts/local.mk
           functionalTestFiles
diff --git a/mk/common-test.sh b/mk/common-test.sh
index 00ccd1584..2783d293b 100644
--- a/mk/common-test.sh
+++ b/mk/common-test.sh
@@ -1,7 +1,7 @@
 # Remove overall test dir (at most one of the two should match) and
 # remove file extension.
 test_name=$(echo -n "$test" | sed \
-    -e "s|^unit-test-data/||" \
+    -e "s|^tests/unit/[^/]*/data/||" \
     -e "s|^tests/functional/||" \
     -e "s|\.sh$||" \
     )
diff --git a/mk/programs.mk b/mk/programs.mk
index a88d9d949..6235311e9 100644
--- a/mk/programs.mk
+++ b/mk/programs.mk
@@ -87,6 +87,6 @@ define build-program
   # Phony target to run this program (typically as a dependency of 'check').
   .PHONY: $(1)_RUN
   $(1)_RUN: $$($(1)_PATH)
-	$(trace-test) $$(UNIT_TEST_ENV) $$($(1)_PATH)
+	$(trace-test) $$($(1)_ENV) $$($(1)_PATH)
 
 endef
diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk
deleted file mode 100644
index 7689a03e0..000000000
--- a/src/libexpr/tests/local.mk
+++ /dev/null
@@ -1,23 +0,0 @@
-check: libexpr-tests_RUN
-
-programs += libexpr-tests
-
-libexpr-tests_NAME := libnixexpr-tests
-
-libexpr-tests_DIR := $(d)
-
-ifeq ($(INSTALL_UNIT_TESTS), yes)
-  libexpr-tests_INSTALL_DIR := $(checkbindir)
-else
-  libexpr-tests_INSTALL_DIR :=
-endif
-
-libexpr-tests_SOURCES := \
-    $(wildcard $(d)/*.cc) \
-    $(wildcard $(d)/value/*.cc)
-
-libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -I src/libfetchers
-
-libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers
-
-libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock
diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk
deleted file mode 100644
index e9b8b4f99..000000000
--- a/src/libstore/tests/local.mk
+++ /dev/null
@@ -1,37 +0,0 @@
-check: libstore-tests-exe_RUN
-
-programs += libstore-tests-exe
-
-libstore-tests-exe_NAME = libnixstore-tests
-
-libstore-tests-exe_DIR := $(d)
-
-ifeq ($(INSTALL_UNIT_TESTS), yes)
-  libstore-tests-exe_INSTALL_DIR := $(checkbindir)
-else
-  libstore-tests-exe_INSTALL_DIR :=
-endif
-
-libstore-tests-exe_LIBS = libstore-tests
-
-libstore-tests-exe_LDFLAGS := $(GTEST_LIBS)
-
-libraries += libstore-tests
-
-libstore-tests_NAME = libnixstore-tests
-
-libstore-tests_DIR := $(d)
-
-ifeq ($(INSTALL_UNIT_TESTS), yes)
-  libstore-tests_INSTALL_DIR := $(checklibdir)
-else
-  libstore-tests_INSTALL_DIR :=
-endif
-
-libstore-tests_SOURCES := $(wildcard $(d)/*.cc)
-
-libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil
-
-libstore-tests_LIBS = libutil-tests libstore libutil
-
-libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS)
diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk
deleted file mode 100644
index 66886c45f..000000000
--- a/src/libutil/tests/local.mk
+++ /dev/null
@@ -1,41 +0,0 @@
-check: libutil-tests-exe_RUN
-
-programs += libutil-tests-exe
-
-libutil-tests-exe_NAME = libnixutil-tests
-
-libutil-tests-exe_DIR := $(d)
-
-ifeq ($(INSTALL_UNIT_TESTS), yes)
-  libutil-tests-exe_INSTALL_DIR := $(checkbindir)
-else
-  libutil-tests-exe_INSTALL_DIR :=
-endif
-
-libutil-tests-exe_LIBS = libutil-tests
-
-libutil-tests-exe_LDFLAGS := $(GTEST_LIBS)
-
-libraries += libutil-tests
-
-libutil-tests_NAME = libnixutil-tests
-
-libutil-tests_DIR := $(d)
-
-ifeq ($(INSTALL_UNIT_TESTS), yes)
-  libutil-tests_INSTALL_DIR := $(checklibdir)
-else
-  libutil-tests_INSTALL_DIR :=
-endif
-
-libutil-tests_SOURCES := $(wildcard $(d)/*.cc)
-
-libutil-tests_CXXFLAGS += -I src/libutil
-
-libutil-tests_LIBS = libutil
-
-libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS)
-
-check: unit-test-data/libutil/git/check-data.sh.test
-
-$(eval $(call run-test,unit-test-data/libutil/git/check-data.sh))
diff --git a/tests/unit/libexpr-support/local.mk b/tests/unit/libexpr-support/local.mk
new file mode 100644
index 000000000..28e87b8f2
--- /dev/null
+++ b/tests/unit/libexpr-support/local.mk
@@ -0,0 +1,23 @@
+libraries += libexpr-test-support
+
+libexpr-test-support_NAME = libnixexpr-test-support
+
+libexpr-test-support_DIR := $(d)
+
+ifeq ($(INSTALL_UNIT_TESTS), yes)
+  libexpr-test-support_INSTALL_DIR := $(checklibdir)
+else
+  libexpr-test-support_INSTALL_DIR :=
+endif
+
+libexpr-test-support_SOURCES := \
+    $(wildcard $(d)/tests/*.cc) \
+    $(wildcard $(d)/tests/value/*.cc)
+
+libexpr-test-support_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES)
+
+libexpr-test-support_LIBS = \
+    libstore-test-support libutil-test-support \
+    libexpr libstore libutil
+
+libexpr-test-support_LDFLAGS := -lrapidcheck
diff --git a/src/libexpr/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh
similarity index 100%
rename from src/libexpr/tests/libexpr.hh
rename to tests/unit/libexpr-support/tests/libexpr.hh
diff --git a/tests/unit/libexpr-support/tests/value/context.cc b/tests/unit/libexpr-support/tests/value/context.cc
new file mode 100644
index 000000000..8658bdaef
--- /dev/null
+++ b/tests/unit/libexpr-support/tests/value/context.cc
@@ -0,0 +1,30 @@
+#include <rapidcheck.h>
+
+#include "tests/path.hh"
+#include "tests/value/context.hh"
+
+namespace rc {
+using namespace nix;
+
+Gen<NixStringContextElem::DrvDeep> Arbitrary<NixStringContextElem::DrvDeep>::arbitrary()
+{
+    return gen::just(NixStringContextElem::DrvDeep {
+        .drvPath = *gen::arbitrary<StorePath>(),
+    });
+}
+
+Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary()
+{
+    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<NixStringContextElem::Raw>)) {
+    case 0:
+        return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Opaque>());
+    case 1:
+        return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::DrvDeep>());
+    case 2:
+        return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Built>());
+    default:
+        assert(false);
+    }
+}
+
+}
diff --git a/src/libexpr/tests/value/context.hh b/tests/unit/libexpr-support/tests/value/context.hh
similarity index 95%
rename from src/libexpr/tests/value/context.hh
rename to tests/unit/libexpr-support/tests/value/context.hh
index c0bc97ba3..8c68c78bb 100644
--- a/src/libexpr/tests/value/context.hh
+++ b/tests/unit/libexpr-support/tests/value/context.hh
@@ -3,7 +3,7 @@
 
 #include <rapidcheck/gen/Arbitrary.h>
 
-#include <value/context.hh>
+#include "value/context.hh"
 
 namespace rc {
 using namespace nix;
diff --git a/src/libexpr/tests/derived-path.cc b/tests/unit/libexpr/derived-path.cc
similarity index 100%
rename from src/libexpr/tests/derived-path.cc
rename to tests/unit/libexpr/derived-path.cc
diff --git a/src/libexpr/tests/error_traces.cc b/tests/unit/libexpr/error_traces.cc
similarity index 100%
rename from src/libexpr/tests/error_traces.cc
rename to tests/unit/libexpr/error_traces.cc
diff --git a/src/libexpr/tests/flakeref.cc b/tests/unit/libexpr/flakeref.cc
similarity index 100%
rename from src/libexpr/tests/flakeref.cc
rename to tests/unit/libexpr/flakeref.cc
diff --git a/src/libexpr/tests/json.cc b/tests/unit/libexpr/json.cc
similarity index 100%
rename from src/libexpr/tests/json.cc
rename to tests/unit/libexpr/json.cc
diff --git a/tests/unit/libexpr/local.mk b/tests/unit/libexpr/local.mk
new file mode 100644
index 000000000..5743880d7
--- /dev/null
+++ b/tests/unit/libexpr/local.mk
@@ -0,0 +1,36 @@
+check: libexpr-tests_RUN
+
+programs += libexpr-tests
+
+libexpr-tests_NAME := libnixexpr-tests
+
+libexpr-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data
+
+libexpr-tests_DIR := $(d)
+
+ifeq ($(INSTALL_UNIT_TESTS), yes)
+  libexpr-tests_INSTALL_DIR := $(checkbindir)
+else
+  libexpr-tests_INSTALL_DIR :=
+endif
+
+libexpr-tests_SOURCES := \
+    $(wildcard $(d)/*.cc) \
+    $(wildcard $(d)/value/*.cc)
+
+libexpr-tests_EXTRA_INCLUDES = \
+    -I tests/unit/libexpr-support \
+    -I tests/unit/libstore-support \
+    -I tests/unit/libutil-support \
+    -I src/libexpr \
+    -I src/libfetchers \
+    -I src/libstore \
+    -I src/libutil
+
+libexpr-tests_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES)
+
+libexpr-tests_LIBS = \
+    libexpr-test-support libstore-test-support libutils-test-support \
+    libexpr libfetchers libstore libutil
+
+libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock
diff --git a/src/libexpr/tests/primops.cc b/tests/unit/libexpr/primops.cc
similarity index 100%
rename from src/libexpr/tests/primops.cc
rename to tests/unit/libexpr/primops.cc
diff --git a/src/libexpr/tests/search-path.cc b/tests/unit/libexpr/search-path.cc
similarity index 100%
rename from src/libexpr/tests/search-path.cc
rename to tests/unit/libexpr/search-path.cc
diff --git a/src/libexpr/tests/trivial.cc b/tests/unit/libexpr/trivial.cc
similarity index 100%
rename from src/libexpr/tests/trivial.cc
rename to tests/unit/libexpr/trivial.cc
diff --git a/src/libexpr/tests/value/context.cc b/tests/unit/libexpr/value/context.cc
similarity index 83%
rename from src/libexpr/tests/value/context.cc
rename to tests/unit/libexpr/value/context.cc
index 92d4889ab..761286dbd 100644
--- a/src/libexpr/tests/value/context.cc
+++ b/tests/unit/libexpr/value/context.cc
@@ -117,36 +117,6 @@ TEST(NixStringContextElemTest, built_built_xp) {
         NixStringContextElem::parse("!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"),        MissingExperimentalFeature);
 }
 
-}
-
-namespace rc {
-using namespace nix;
-
-Gen<NixStringContextElem::DrvDeep> Arbitrary<NixStringContextElem::DrvDeep>::arbitrary()
-{
-    return gen::just(NixStringContextElem::DrvDeep {
-        .drvPath = *gen::arbitrary<StorePath>(),
-    });
-}
-
-Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary()
-{
-    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<NixStringContextElem::Raw>)) {
-    case 0:
-        return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Opaque>());
-    case 1:
-        return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::DrvDeep>());
-    case 2:
-        return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Built>());
-    default:
-        assert(false);
-    }
-}
-
-}
-
-namespace nix {
-
 #ifndef COVERAGE
 
 RC_GTEST_PROP(
diff --git a/src/libexpr/tests/value/print.cc b/tests/unit/libexpr/value/print.cc
similarity index 100%
rename from src/libexpr/tests/value/print.cc
rename to tests/unit/libexpr/value/print.cc
diff --git a/tests/unit/libstore-support/local.mk b/tests/unit/libstore-support/local.mk
new file mode 100644
index 000000000..d5d657c91
--- /dev/null
+++ b/tests/unit/libstore-support/local.mk
@@ -0,0 +1,21 @@
+libraries += libstore-test-support
+
+libstore-test-support_NAME = libnixstore-test-support
+
+libstore-test-support_DIR := $(d)
+
+ifeq ($(INSTALL_UNIT_TESTS), yes)
+  libstore-test-support_INSTALL_DIR := $(checklibdir)
+else
+  libstore-test-support_INSTALL_DIR :=
+endif
+
+libstore-test-support_SOURCES := $(wildcard $(d)/tests/*.cc)
+
+libstore-test-support_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES)
+
+libstore-test-support_LIBS = \
+    libutil-test-support \
+    libstore libutil
+
+libstore-test-support_LDFLAGS := -lrapidcheck
diff --git a/tests/unit/libstore-support/tests/derived-path.cc b/tests/unit/libstore-support/tests/derived-path.cc
new file mode 100644
index 000000000..091706dba
--- /dev/null
+++ b/tests/unit/libstore-support/tests/derived-path.cc
@@ -0,0 +1,57 @@
+#include <regex>
+
+#include <rapidcheck.h>
+
+#include "tests/derived-path.hh"
+
+namespace rc {
+using namespace nix;
+
+Gen<DerivedPath::Opaque> Arbitrary<DerivedPath::Opaque>::arbitrary()
+{
+    return gen::just(DerivedPath::Opaque {
+        .path = *gen::arbitrary<StorePath>(),
+    });
+}
+
+Gen<SingleDerivedPath::Built> Arbitrary<SingleDerivedPath::Built>::arbitrary()
+{
+    return gen::just(SingleDerivedPath::Built {
+        .drvPath = make_ref<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath>()),
+        .output = (*gen::arbitrary<StorePathName>()).name,
+    });
+}
+
+Gen<DerivedPath::Built> Arbitrary<DerivedPath::Built>::arbitrary()
+{
+    return gen::just(DerivedPath::Built {
+        .drvPath = make_ref<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath>()),
+        .outputs = *gen::arbitrary<OutputsSpec>(),
+    });
+}
+
+Gen<SingleDerivedPath> Arbitrary<SingleDerivedPath>::arbitrary()
+{
+    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<SingleDerivedPath::Raw>)) {
+    case 0:
+        return gen::just<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath::Opaque>());
+    case 1:
+        return gen::just<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath::Built>());
+    default:
+        assert(false);
+    }
+}
+
+Gen<DerivedPath> Arbitrary<DerivedPath>::arbitrary()
+{
+    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<DerivedPath::Raw>)) {
+    case 0:
+        return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Opaque>());
+    case 1:
+        return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Built>());
+    default:
+        assert(false);
+    }
+}
+
+}
diff --git a/src/libstore/tests/derived-path.hh b/tests/unit/libstore-support/tests/derived-path.hh
similarity index 100%
rename from src/libstore/tests/derived-path.hh
rename to tests/unit/libstore-support/tests/derived-path.hh
diff --git a/src/libstore/tests/libstore.hh b/tests/unit/libstore-support/tests/libstore.hh
similarity index 100%
rename from src/libstore/tests/libstore.hh
rename to tests/unit/libstore-support/tests/libstore.hh
diff --git a/tests/unit/libstore-support/tests/outputs-spec.cc b/tests/unit/libstore-support/tests/outputs-spec.cc
new file mode 100644
index 000000000..e9d602203
--- /dev/null
+++ b/tests/unit/libstore-support/tests/outputs-spec.cc
@@ -0,0 +1,24 @@
+#include "tests/outputs-spec.hh"
+
+#include <rapidcheck.h>
+
+namespace rc {
+using namespace nix;
+
+Gen<OutputsSpec> Arbitrary<OutputsSpec>::arbitrary()
+{
+    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<OutputsSpec::Raw>)) {
+    case 0:
+        return gen::just((OutputsSpec) OutputsSpec::All { });
+    case 1:
+        return gen::just((OutputsSpec) OutputsSpec::Names {
+            *gen::nonEmpty(gen::container<StringSet>(gen::map(
+                gen::arbitrary<StorePathName>(),
+                [](StorePathName n) { return n.name; }))),
+        });
+    default:
+        assert(false);
+    }
+}
+
+}
diff --git a/src/libstore/tests/outputs-spec.hh b/tests/unit/libstore-support/tests/outputs-spec.hh
similarity index 89%
rename from src/libstore/tests/outputs-spec.hh
rename to tests/unit/libstore-support/tests/outputs-spec.hh
index ded331b33..f5bf9042d 100644
--- a/src/libstore/tests/outputs-spec.hh
+++ b/tests/unit/libstore-support/tests/outputs-spec.hh
@@ -5,7 +5,7 @@
 
 #include <outputs-spec.hh>
 
-#include <tests/path.hh>
+#include "tests/path.hh"
 
 namespace rc {
 using namespace nix;
diff --git a/tests/unit/libstore-support/tests/path.cc b/tests/unit/libstore-support/tests/path.cc
new file mode 100644
index 000000000..e5f169e94
--- /dev/null
+++ b/tests/unit/libstore-support/tests/path.cc
@@ -0,0 +1,82 @@
+#include <regex>
+
+#include <rapidcheck.h>
+
+#include "path-regex.hh"
+#include "store-api.hh"
+
+#include "tests/hash.hh"
+#include "tests/path.hh"
+
+namespace nix {
+
+void showValue(const StorePath & p, std::ostream & os)
+{
+    os << p.to_string();
+}
+
+}
+
+namespace rc {
+using namespace nix;
+
+Gen<StorePathName> Arbitrary<StorePathName>::arbitrary()
+{
+    auto len = *gen::inRange<size_t>(
+        1,
+        StorePath::MaxPathLen - StorePath::HashLen);
+
+    std::string pre;
+    pre.reserve(len);
+
+    for (size_t c = 0; c < len; ++c) {
+        switch (auto i = *gen::inRange<uint8_t>(0, 10 + 2 * 26 + 6)) {
+            case 0 ... 9:
+                pre += '0' + i;
+            case 10 ... 35:
+                pre += 'A' + (i - 10);
+                break;
+            case 36 ... 61:
+                pre += 'a' + (i - 36);
+                break;
+            case 62:
+                pre += '+';
+                break;
+            case 63:
+                pre += '-';
+                break;
+            case 64:
+                // names aren't permitted to start with a period,
+                // so just fall through to the next case here
+                if (c != 0) {
+                    pre += '.';
+                    break;
+                }
+            case 65:
+                pre += '_';
+                break;
+            case 66:
+                pre += '?';
+                break;
+            case 67:
+                pre += '=';
+                break;
+            default:
+                assert(false);
+        }
+    }
+
+    return gen::just(StorePathName {
+        .name = std::move(pre),
+    });
+}
+
+Gen<StorePath> Arbitrary<StorePath>::arbitrary()
+{
+    return gen::just(StorePath {
+        *gen::arbitrary<Hash>(),
+        (*gen::arbitrary<StorePathName>()).name,
+    });
+}
+
+} // namespace rc
diff --git a/src/libstore/tests/path.hh b/tests/unit/libstore-support/tests/path.hh
similarity index 82%
rename from src/libstore/tests/path.hh
rename to tests/unit/libstore-support/tests/path.hh
index 21cb62310..4751b3373 100644
--- a/src/libstore/tests/path.hh
+++ b/tests/unit/libstore-support/tests/path.hh
@@ -11,6 +11,9 @@ struct StorePathName {
     std::string name;
 };
 
+// For rapidcheck
+void showValue(const StorePath & p, std::ostream & os);
+
 }
 
 namespace rc {
diff --git a/src/libstore/tests/protocol.hh b/tests/unit/libstore-support/tests/protocol.hh
similarity index 96%
rename from src/libstore/tests/protocol.hh
rename to tests/unit/libstore-support/tests/protocol.hh
index 466032a79..3c9e52c11 100644
--- a/src/libstore/tests/protocol.hh
+++ b/tests/unit/libstore-support/tests/protocol.hh
@@ -12,7 +12,7 @@ namespace nix {
 template<class Proto, const char * protocolDir>
 class ProtoTest : public CharacterizationTest, public LibStoreTest
 {
-    Path unitTestData = getUnitTestData() + "/libstore/" + protocolDir;
+    Path unitTestData = getUnitTestData() + "/" + protocolDir;
 
     Path goldenMaster(std::string_view testStem) const override {
         return unitTestData + "/" + testStem + ".bin";
diff --git a/src/libstore/tests/common-protocol.cc b/tests/unit/libstore/common-protocol.cc
similarity index 100%
rename from src/libstore/tests/common-protocol.cc
rename to tests/unit/libstore/common-protocol.cc
diff --git a/unit-test-data/libstore/common-protocol/content-address.bin b/tests/unit/libstore/data/common-protocol/content-address.bin
similarity index 100%
rename from unit-test-data/libstore/common-protocol/content-address.bin
rename to tests/unit/libstore/data/common-protocol/content-address.bin
diff --git a/unit-test-data/libstore/common-protocol/drv-output.bin b/tests/unit/libstore/data/common-protocol/drv-output.bin
similarity index 100%
rename from unit-test-data/libstore/common-protocol/drv-output.bin
rename to tests/unit/libstore/data/common-protocol/drv-output.bin
diff --git a/unit-test-data/libstore/common-protocol/optional-content-address.bin b/tests/unit/libstore/data/common-protocol/optional-content-address.bin
similarity index 100%
rename from unit-test-data/libstore/common-protocol/optional-content-address.bin
rename to tests/unit/libstore/data/common-protocol/optional-content-address.bin
diff --git a/unit-test-data/libstore/common-protocol/optional-store-path.bin b/tests/unit/libstore/data/common-protocol/optional-store-path.bin
similarity index 100%
rename from unit-test-data/libstore/common-protocol/optional-store-path.bin
rename to tests/unit/libstore/data/common-protocol/optional-store-path.bin
diff --git a/unit-test-data/libstore/common-protocol/realisation.bin b/tests/unit/libstore/data/common-protocol/realisation.bin
similarity index 100%
rename from unit-test-data/libstore/common-protocol/realisation.bin
rename to tests/unit/libstore/data/common-protocol/realisation.bin
diff --git a/unit-test-data/libstore/common-protocol/set.bin b/tests/unit/libstore/data/common-protocol/set.bin
similarity index 100%
rename from unit-test-data/libstore/common-protocol/set.bin
rename to tests/unit/libstore/data/common-protocol/set.bin
diff --git a/unit-test-data/libstore/common-protocol/store-path.bin b/tests/unit/libstore/data/common-protocol/store-path.bin
similarity index 100%
rename from unit-test-data/libstore/common-protocol/store-path.bin
rename to tests/unit/libstore/data/common-protocol/store-path.bin
diff --git a/unit-test-data/libstore/common-protocol/string.bin b/tests/unit/libstore/data/common-protocol/string.bin
similarity index 100%
rename from unit-test-data/libstore/common-protocol/string.bin
rename to tests/unit/libstore/data/common-protocol/string.bin
diff --git a/unit-test-data/libstore/common-protocol/vector.bin b/tests/unit/libstore/data/common-protocol/vector.bin
similarity index 100%
rename from unit-test-data/libstore/common-protocol/vector.bin
rename to tests/unit/libstore/data/common-protocol/vector.bin
diff --git a/unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv b/tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv
similarity index 100%
rename from unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv
rename to tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv
diff --git a/unit-test-data/libstore/derivation/bad-version.drv b/tests/unit/libstore/data/derivation/bad-version.drv
similarity index 100%
rename from unit-test-data/libstore/derivation/bad-version.drv
rename to tests/unit/libstore/data/derivation/bad-version.drv
diff --git a/unit-test-data/libstore/derivation/dynDerivationDeps.drv b/tests/unit/libstore/data/derivation/dynDerivationDeps.drv
similarity index 100%
rename from unit-test-data/libstore/derivation/dynDerivationDeps.drv
rename to tests/unit/libstore/data/derivation/dynDerivationDeps.drv
diff --git a/unit-test-data/libstore/derivation/dynDerivationDeps.json b/tests/unit/libstore/data/derivation/dynDerivationDeps.json
similarity index 100%
rename from unit-test-data/libstore/derivation/dynDerivationDeps.json
rename to tests/unit/libstore/data/derivation/dynDerivationDeps.json
diff --git a/unit-test-data/libstore/derivation/output-caFixedFlat.json b/tests/unit/libstore/data/derivation/output-caFixedFlat.json
similarity index 100%
rename from unit-test-data/libstore/derivation/output-caFixedFlat.json
rename to tests/unit/libstore/data/derivation/output-caFixedFlat.json
diff --git a/unit-test-data/libstore/derivation/output-caFixedNAR.json b/tests/unit/libstore/data/derivation/output-caFixedNAR.json
similarity index 100%
rename from unit-test-data/libstore/derivation/output-caFixedNAR.json
rename to tests/unit/libstore/data/derivation/output-caFixedNAR.json
diff --git a/unit-test-data/libstore/derivation/output-caFixedText.json b/tests/unit/libstore/data/derivation/output-caFixedText.json
similarity index 100%
rename from unit-test-data/libstore/derivation/output-caFixedText.json
rename to tests/unit/libstore/data/derivation/output-caFixedText.json
diff --git a/unit-test-data/libstore/derivation/output-caFloating.json b/tests/unit/libstore/data/derivation/output-caFloating.json
similarity index 100%
rename from unit-test-data/libstore/derivation/output-caFloating.json
rename to tests/unit/libstore/data/derivation/output-caFloating.json
diff --git a/unit-test-data/libstore/derivation/output-deferred.json b/tests/unit/libstore/data/derivation/output-deferred.json
similarity index 100%
rename from unit-test-data/libstore/derivation/output-deferred.json
rename to tests/unit/libstore/data/derivation/output-deferred.json
diff --git a/unit-test-data/libstore/derivation/output-impure.json b/tests/unit/libstore/data/derivation/output-impure.json
similarity index 100%
rename from unit-test-data/libstore/derivation/output-impure.json
rename to tests/unit/libstore/data/derivation/output-impure.json
diff --git a/unit-test-data/libstore/derivation/output-inputAddressed.json b/tests/unit/libstore/data/derivation/output-inputAddressed.json
similarity index 100%
rename from unit-test-data/libstore/derivation/output-inputAddressed.json
rename to tests/unit/libstore/data/derivation/output-inputAddressed.json
diff --git a/unit-test-data/libstore/derivation/simple.drv b/tests/unit/libstore/data/derivation/simple.drv
similarity index 100%
rename from unit-test-data/libstore/derivation/simple.drv
rename to tests/unit/libstore/data/derivation/simple.drv
diff --git a/unit-test-data/libstore/derivation/simple.json b/tests/unit/libstore/data/derivation/simple.json
similarity index 100%
rename from unit-test-data/libstore/derivation/simple.json
rename to tests/unit/libstore/data/derivation/simple.json
diff --git a/unit-test-data/libstore/nar-info/impure.json b/tests/unit/libstore/data/nar-info/impure.json
similarity index 100%
rename from unit-test-data/libstore/nar-info/impure.json
rename to tests/unit/libstore/data/nar-info/impure.json
diff --git a/unit-test-data/libstore/nar-info/pure.json b/tests/unit/libstore/data/nar-info/pure.json
similarity index 100%
rename from unit-test-data/libstore/nar-info/pure.json
rename to tests/unit/libstore/data/nar-info/pure.json
diff --git a/unit-test-data/libstore/path-info/impure.json b/tests/unit/libstore/data/path-info/impure.json
similarity index 100%
rename from unit-test-data/libstore/path-info/impure.json
rename to tests/unit/libstore/data/path-info/impure.json
diff --git a/unit-test-data/libstore/path-info/pure.json b/tests/unit/libstore/data/path-info/pure.json
similarity index 100%
rename from unit-test-data/libstore/path-info/pure.json
rename to tests/unit/libstore/data/path-info/pure.json
diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.2.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.2.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/build-result-2.2.bin
rename to tests/unit/libstore/data/serve-protocol/build-result-2.2.bin
diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.3.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.3.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/build-result-2.3.bin
rename to tests/unit/libstore/data/serve-protocol/build-result-2.3.bin
diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.6.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.6.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/build-result-2.6.bin
rename to tests/unit/libstore/data/serve-protocol/build-result-2.6.bin
diff --git a/unit-test-data/libstore/serve-protocol/content-address.bin b/tests/unit/libstore/data/serve-protocol/content-address.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/content-address.bin
rename to tests/unit/libstore/data/serve-protocol/content-address.bin
diff --git a/unit-test-data/libstore/serve-protocol/drv-output.bin b/tests/unit/libstore/data/serve-protocol/drv-output.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/drv-output.bin
rename to tests/unit/libstore/data/serve-protocol/drv-output.bin
diff --git a/unit-test-data/libstore/serve-protocol/optional-content-address.bin b/tests/unit/libstore/data/serve-protocol/optional-content-address.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/optional-content-address.bin
rename to tests/unit/libstore/data/serve-protocol/optional-content-address.bin
diff --git a/unit-test-data/libstore/serve-protocol/optional-store-path.bin b/tests/unit/libstore/data/serve-protocol/optional-store-path.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/optional-store-path.bin
rename to tests/unit/libstore/data/serve-protocol/optional-store-path.bin
diff --git a/unit-test-data/libstore/serve-protocol/realisation.bin b/tests/unit/libstore/data/serve-protocol/realisation.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/realisation.bin
rename to tests/unit/libstore/data/serve-protocol/realisation.bin
diff --git a/unit-test-data/libstore/serve-protocol/set.bin b/tests/unit/libstore/data/serve-protocol/set.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/set.bin
rename to tests/unit/libstore/data/serve-protocol/set.bin
diff --git a/unit-test-data/libstore/serve-protocol/store-path.bin b/tests/unit/libstore/data/serve-protocol/store-path.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/store-path.bin
rename to tests/unit/libstore/data/serve-protocol/store-path.bin
diff --git a/unit-test-data/libstore/serve-protocol/string.bin b/tests/unit/libstore/data/serve-protocol/string.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/string.bin
rename to tests/unit/libstore/data/serve-protocol/string.bin
diff --git a/unit-test-data/libstore/serve-protocol/vector.bin b/tests/unit/libstore/data/serve-protocol/vector.bin
similarity index 100%
rename from unit-test-data/libstore/serve-protocol/vector.bin
rename to tests/unit/libstore/data/serve-protocol/vector.bin
diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.27.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.27.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/build-result-1.27.bin
rename to tests/unit/libstore/data/worker-protocol/build-result-1.27.bin
diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.28.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.28.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/build-result-1.28.bin
rename to tests/unit/libstore/data/worker-protocol/build-result-1.28.bin
diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.29.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.29.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/build-result-1.29.bin
rename to tests/unit/libstore/data/worker-protocol/build-result-1.29.bin
diff --git a/unit-test-data/libstore/worker-protocol/content-address.bin b/tests/unit/libstore/data/worker-protocol/content-address.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/content-address.bin
rename to tests/unit/libstore/data/worker-protocol/content-address.bin
diff --git a/unit-test-data/libstore/worker-protocol/derived-path-1.29.bin b/tests/unit/libstore/data/worker-protocol/derived-path-1.29.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/derived-path-1.29.bin
rename to tests/unit/libstore/data/worker-protocol/derived-path-1.29.bin
diff --git a/unit-test-data/libstore/worker-protocol/derived-path-1.30.bin b/tests/unit/libstore/data/worker-protocol/derived-path-1.30.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/derived-path-1.30.bin
rename to tests/unit/libstore/data/worker-protocol/derived-path-1.30.bin
diff --git a/unit-test-data/libstore/worker-protocol/drv-output.bin b/tests/unit/libstore/data/worker-protocol/drv-output.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/drv-output.bin
rename to tests/unit/libstore/data/worker-protocol/drv-output.bin
diff --git a/unit-test-data/libstore/worker-protocol/keyed-build-result-1.29.bin b/tests/unit/libstore/data/worker-protocol/keyed-build-result-1.29.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/keyed-build-result-1.29.bin
rename to tests/unit/libstore/data/worker-protocol/keyed-build-result-1.29.bin
diff --git a/unit-test-data/libstore/worker-protocol/optional-content-address.bin b/tests/unit/libstore/data/worker-protocol/optional-content-address.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/optional-content-address.bin
rename to tests/unit/libstore/data/worker-protocol/optional-content-address.bin
diff --git a/unit-test-data/libstore/worker-protocol/optional-store-path.bin b/tests/unit/libstore/data/worker-protocol/optional-store-path.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/optional-store-path.bin
rename to tests/unit/libstore/data/worker-protocol/optional-store-path.bin
diff --git a/unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin b/tests/unit/libstore/data/worker-protocol/optional-trusted-flag.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin
rename to tests/unit/libstore/data/worker-protocol/optional-trusted-flag.bin
diff --git a/unit-test-data/libstore/worker-protocol/realisation.bin b/tests/unit/libstore/data/worker-protocol/realisation.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/realisation.bin
rename to tests/unit/libstore/data/worker-protocol/realisation.bin
diff --git a/unit-test-data/libstore/worker-protocol/set.bin b/tests/unit/libstore/data/worker-protocol/set.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/set.bin
rename to tests/unit/libstore/data/worker-protocol/set.bin
diff --git a/unit-test-data/libstore/worker-protocol/store-path.bin b/tests/unit/libstore/data/worker-protocol/store-path.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/store-path.bin
rename to tests/unit/libstore/data/worker-protocol/store-path.bin
diff --git a/unit-test-data/libstore/worker-protocol/string.bin b/tests/unit/libstore/data/worker-protocol/string.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/string.bin
rename to tests/unit/libstore/data/worker-protocol/string.bin
diff --git a/unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin b/tests/unit/libstore/data/worker-protocol/unkeyed-valid-path-info-1.15.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin
rename to tests/unit/libstore/data/worker-protocol/unkeyed-valid-path-info-1.15.bin
diff --git a/unit-test-data/libstore/worker-protocol/valid-path-info-1.15.bin b/tests/unit/libstore/data/worker-protocol/valid-path-info-1.15.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/valid-path-info-1.15.bin
rename to tests/unit/libstore/data/worker-protocol/valid-path-info-1.15.bin
diff --git a/unit-test-data/libstore/worker-protocol/valid-path-info-1.16.bin b/tests/unit/libstore/data/worker-protocol/valid-path-info-1.16.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/valid-path-info-1.16.bin
rename to tests/unit/libstore/data/worker-protocol/valid-path-info-1.16.bin
diff --git a/unit-test-data/libstore/worker-protocol/vector.bin b/tests/unit/libstore/data/worker-protocol/vector.bin
similarity index 100%
rename from unit-test-data/libstore/worker-protocol/vector.bin
rename to tests/unit/libstore/data/worker-protocol/vector.bin
diff --git a/src/libstore/tests/derivation.cc b/tests/unit/libstore/derivation.cc
similarity index 99%
rename from src/libstore/tests/derivation.cc
rename to tests/unit/libstore/derivation.cc
index 7becfa5ab..a7f4488fa 100644
--- a/src/libstore/tests/derivation.cc
+++ b/tests/unit/libstore/derivation.cc
@@ -13,7 +13,7 @@ using nlohmann::json;
 
 class DerivationTest : public CharacterizationTest, public LibStoreTest
 {
-    Path unitTestData = getUnitTestData() + "/libstore/derivation";
+    Path unitTestData = getUnitTestData() + "/derivation";
 
 public:
     Path goldenMaster(std::string_view testStem) const override {
diff --git a/src/libstore/tests/derived-path.cc b/tests/unit/libstore/derived-path.cc
similarity index 66%
rename from src/libstore/tests/derived-path.cc
rename to tests/unit/libstore/derived-path.cc
index 3fa3c0801..c62d79a78 100644
--- a/src/libstore/tests/derived-path.cc
+++ b/tests/unit/libstore/derived-path.cc
@@ -1,64 +1,11 @@
 #include <regex>
 
-#include <nlohmann/json.hpp>
 #include <gtest/gtest.h>
 #include <rapidcheck/gtest.h>
 
 #include "tests/derived-path.hh"
 #include "tests/libstore.hh"
 
-namespace rc {
-using namespace nix;
-
-Gen<DerivedPath::Opaque> Arbitrary<DerivedPath::Opaque>::arbitrary()
-{
-    return gen::just(DerivedPath::Opaque {
-        .path = *gen::arbitrary<StorePath>(),
-    });
-}
-
-Gen<SingleDerivedPath::Built> Arbitrary<SingleDerivedPath::Built>::arbitrary()
-{
-    return gen::just(SingleDerivedPath::Built {
-        .drvPath = make_ref<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath>()),
-        .output = (*gen::arbitrary<StorePathName>()).name,
-    });
-}
-
-Gen<DerivedPath::Built> Arbitrary<DerivedPath::Built>::arbitrary()
-{
-    return gen::just(DerivedPath::Built {
-        .drvPath = make_ref<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath>()),
-        .outputs = *gen::arbitrary<OutputsSpec>(),
-    });
-}
-
-Gen<SingleDerivedPath> Arbitrary<SingleDerivedPath>::arbitrary()
-{
-    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<SingleDerivedPath::Raw>)) {
-    case 0:
-        return gen::just<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath::Opaque>());
-    case 1:
-        return gen::just<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath::Built>());
-    default:
-        assert(false);
-    }
-}
-
-Gen<DerivedPath> Arbitrary<DerivedPath>::arbitrary()
-{
-    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<DerivedPath::Raw>)) {
-    case 0:
-        return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Opaque>());
-    case 1:
-        return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Built>());
-    default:
-        assert(false);
-    }
-}
-
-}
-
 namespace nix {
 
 class DerivedPathTest : public LibStoreTest
diff --git a/src/libstore/tests/downstream-placeholder.cc b/tests/unit/libstore/downstream-placeholder.cc
similarity index 100%
rename from src/libstore/tests/downstream-placeholder.cc
rename to tests/unit/libstore/downstream-placeholder.cc
diff --git a/tests/unit/libstore/local.mk b/tests/unit/libstore/local.mk
new file mode 100644
index 000000000..63f6d011f
--- /dev/null
+++ b/tests/unit/libstore/local.mk
@@ -0,0 +1,31 @@
+check: libstore-tests_RUN
+
+programs += libstore-tests
+
+libstore-tests_NAME = libnixstore-tests
+
+libstore-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data
+
+libstore-tests_DIR := $(d)
+
+ifeq ($(INSTALL_UNIT_TESTS), yes)
+  libstore-tests_INSTALL_DIR := $(checkbindir)
+else
+  libstore-tests_INSTALL_DIR :=
+endif
+
+libstore-tests_SOURCES := $(wildcard $(d)/*.cc)
+
+libstore-tests_EXTRA_INCLUDES = \
+    -I tests/unit/libstore-support \
+    -I tests/unit/libutil-support \
+    -I src/libstore \
+    -I src/libutil
+
+libstore-tests_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES)
+
+libstore-tests_LIBS = \
+    libstore-test-support libutil-test-support \
+    libstore libutil
+
+libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS)
diff --git a/src/libstore/tests/machines.cc b/tests/unit/libstore/machines.cc
similarity index 97%
rename from src/libstore/tests/machines.cc
rename to tests/unit/libstore/machines.cc
index fede328ea..5b66e5a5b 100644
--- a/src/libstore/tests/machines.cc
+++ b/tests/unit/libstore/machines.cc
@@ -139,7 +139,7 @@ TEST(machines, getMachinesWithIncorrectFormat) {
 }
 
 TEST(machines, getMachinesWithCorrectFileReference) {
-    auto path = absPath("src/libstore/tests/test-data/machines.valid");
+    auto path = absPath("tests/unit/libstore/test-data/machines.valid");
     ASSERT_TRUE(pathExists(path));
 
     settings.builders = std::string("@") + path;
@@ -166,6 +166,6 @@ TEST(machines, getMachinesWithIncorrectFileReference) {
 }
 
 TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) {
-    settings.builders = std::string("@") + absPath("src/libstore/tests/test-data/machines.bad_format");
+    settings.builders = std::string("@") + absPath("tests/unit/libstore/test-data/machines.bad_format");
     EXPECT_THROW(getMachines(), FormatError);
 }
diff --git a/src/libstore/tests/nar-info-disk-cache.cc b/tests/unit/libstore/nar-info-disk-cache.cc
similarity index 100%
rename from src/libstore/tests/nar-info-disk-cache.cc
rename to tests/unit/libstore/nar-info-disk-cache.cc
diff --git a/src/libstore/tests/nar-info.cc b/tests/unit/libstore/nar-info.cc
similarity index 98%
rename from src/libstore/tests/nar-info.cc
rename to tests/unit/libstore/nar-info.cc
index 88e6e1add..4f124e89e 100644
--- a/src/libstore/tests/nar-info.cc
+++ b/tests/unit/libstore/nar-info.cc
@@ -13,7 +13,7 @@ using nlohmann::json;
 
 class NarInfoTest : public CharacterizationTest, public LibStoreTest
 {
-    Path unitTestData = getUnitTestData() + "/libstore/nar-info";
+    Path unitTestData = getUnitTestData() + "/nar-info";
 
     Path goldenMaster(PathView testStem) const override {
         return unitTestData + "/" + testStem + ".json";
diff --git a/src/libstore/tests/outputs-spec.cc b/tests/unit/libstore/outputs-spec.cc
similarity index 92%
rename from src/libstore/tests/outputs-spec.cc
rename to tests/unit/libstore/outputs-spec.cc
index 952945185..456196be1 100644
--- a/src/libstore/tests/outputs-spec.cc
+++ b/tests/unit/libstore/outputs-spec.cc
@@ -1,4 +1,4 @@
-#include "outputs-spec.hh"
+#include "tests/outputs-spec.hh"
 
 #include <nlohmann/json.hpp>
 #include <gtest/gtest.h>
@@ -199,31 +199,6 @@ TEST_JSON(ExtendedOutputsSpec, names, R"(["a","b"])", (ExtendedOutputsSpec::Expl
 
 #undef TEST_JSON
 
-}
-
-namespace rc {
-using namespace nix;
-
-Gen<OutputsSpec> Arbitrary<OutputsSpec>::arbitrary()
-{
-    switch (*gen::inRange<uint8_t>(0, std::variant_size_v<OutputsSpec::Raw>)) {
-    case 0:
-        return gen::just((OutputsSpec) OutputsSpec::All { });
-    case 1:
-        return gen::just((OutputsSpec) OutputsSpec::Names {
-            *gen::nonEmpty(gen::container<StringSet>(gen::map(
-                gen::arbitrary<StorePathName>(),
-                [](StorePathName n) { return n.name; }))),
-        });
-    default:
-        assert(false);
-    }
-}
-
-}
-
-namespace nix {
-
 #ifndef COVERAGE
 
 RC_GTEST_PROP(
diff --git a/src/libstore/tests/path-info.cc b/tests/unit/libstore/path-info.cc
similarity index 97%
rename from src/libstore/tests/path-info.cc
rename to tests/unit/libstore/path-info.cc
index 49bf623bd..18f00ca19 100644
--- a/src/libstore/tests/path-info.cc
+++ b/tests/unit/libstore/path-info.cc
@@ -12,7 +12,7 @@ using nlohmann::json;
 
 class PathInfoTest : public CharacterizationTest, public LibStoreTest
 {
-    Path unitTestData = getUnitTestData() + "/libstore/path-info";
+    Path unitTestData = getUnitTestData() + "/path-info";
 
     Path goldenMaster(PathView testStem) const override {
         return unitTestData + "/" + testStem + ".json";
diff --git a/src/libstore/tests/path.cc b/tests/unit/libstore/path.cc
similarity index 59%
rename from src/libstore/tests/path.cc
rename to tests/unit/libstore/path.cc
index 5a84d646c..30631b5fd 100644
--- a/src/libstore/tests/path.cc
+++ b/tests/unit/libstore/path.cc
@@ -66,79 +66,6 @@ TEST_DO_PARSE(equals_sign, "foo=foo")
 
 #undef TEST_DO_PARSE
 
-// For rapidcheck
-void showValue(const StorePath & p, std::ostream & os) {
-    os << p.to_string();
-}
-
-}
-
-namespace rc {
-using namespace nix;
-
-Gen<StorePathName> Arbitrary<StorePathName>::arbitrary()
-{
-    auto len = *gen::inRange<size_t>(
-        1,
-        StorePath::MaxPathLen - std::string_view { HASH_PART }.size());
-
-    std::string pre;
-    pre.reserve(len);
-
-    for (size_t c = 0; c < len; ++c) {
-        switch (auto i = *gen::inRange<uint8_t>(0, 10 + 2 * 26 + 6)) {
-            case 0 ... 9:
-                pre += '0' + i;
-            case 10 ... 35:
-                pre += 'A' + (i - 10);
-                break;
-            case 36 ... 61:
-                pre += 'a' + (i - 36);
-                break;
-            case 62:
-                pre += '+';
-                break;
-            case 63:
-                pre += '-';
-                break;
-            case 64:
-                // names aren't permitted to start with a period,
-                // so just fall through to the next case here
-                if (c != 0) {
-                    pre += '.';
-                    break;
-                }
-            case 65:
-                pre += '_';
-                break;
-            case 66:
-                pre += '?';
-                break;
-            case 67:
-                pre += '=';
-                break;
-            default:
-                assert(false);
-        }
-    }
-
-    return gen::just(StorePathName {
-        .name = std::move(pre),
-    });
-}
-
-Gen<StorePath> Arbitrary<StorePath>::arbitrary()
-{
-    return gen::just(StorePath {
-        *gen::arbitrary<Hash>(),
-        (*gen::arbitrary<StorePathName>()).name,
-    });
-}
-
-} // namespace rc
-
-namespace nix {
-
 #ifndef COVERAGE
 
 RC_GTEST_FIXTURE_PROP(
diff --git a/src/libstore/tests/references.cc b/tests/unit/libstore/references.cc
similarity index 100%
rename from src/libstore/tests/references.cc
rename to tests/unit/libstore/references.cc
diff --git a/src/libstore/tests/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc
similarity index 100%
rename from src/libstore/tests/serve-protocol.cc
rename to tests/unit/libstore/serve-protocol.cc
diff --git a/src/libstore/tests/test-data/machines.bad_format b/tests/unit/libstore/test-data/machines.bad_format
similarity index 100%
rename from src/libstore/tests/test-data/machines.bad_format
rename to tests/unit/libstore/test-data/machines.bad_format
diff --git a/src/libstore/tests/test-data/machines.valid b/tests/unit/libstore/test-data/machines.valid
similarity index 100%
rename from src/libstore/tests/test-data/machines.valid
rename to tests/unit/libstore/test-data/machines.valid
diff --git a/src/libstore/tests/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc
similarity index 100%
rename from src/libstore/tests/worker-protocol.cc
rename to tests/unit/libstore/worker-protocol.cc
diff --git a/tests/unit/libutil-support/local.mk b/tests/unit/libutil-support/local.mk
new file mode 100644
index 000000000..43a1551e5
--- /dev/null
+++ b/tests/unit/libutil-support/local.mk
@@ -0,0 +1,19 @@
+libraries += libutil-test-support
+
+libutil-test-support_NAME = libnixutil-test-support
+
+libutil-test-support_DIR := $(d)
+
+ifeq ($(INSTALL_UNIT_TESTS), yes)
+  libutil-test-support_INSTALL_DIR := $(checklibdir)
+else
+  libutil-test-support_INSTALL_DIR :=
+endif
+
+libutil-test-support_SOURCES := $(wildcard $(d)/tests/*.cc)
+
+libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES)
+
+libutil-test-support_LIBS = libutil
+
+libutil-test-support_LDFLAGS := -lrapidcheck
diff --git a/src/libutil/tests/characterization.hh b/tests/unit/libutil-support/tests/characterization.hh
similarity index 95%
rename from src/libutil/tests/characterization.hh
rename to tests/unit/libutil-support/tests/characterization.hh
index 6eb513d68..9d6c850f0 100644
--- a/src/libutil/tests/characterization.hh
+++ b/tests/unit/libutil-support/tests/characterization.hh
@@ -9,8 +9,8 @@
 namespace nix {
 
 /**
- * The path to the `unit-test-data` directory. See the contributing
- * guide in the manual for further details.
+ * The path to the unit test data directory. See the contributing guide
+ * in the manual for further details.
  */
 static Path getUnitTestData() {
     return getEnv("_NIX_TEST_UNIT_DATA").value();
diff --git a/tests/unit/libutil-support/tests/hash.cc b/tests/unit/libutil-support/tests/hash.cc
new file mode 100644
index 000000000..577e9890e
--- /dev/null
+++ b/tests/unit/libutil-support/tests/hash.cc
@@ -0,0 +1,20 @@
+#include <regex>
+
+#include <rapidcheck.h>
+
+#include "hash.hh"
+
+#include "tests/hash.hh"
+
+namespace rc {
+using namespace nix;
+
+Gen<Hash> Arbitrary<Hash>::arbitrary()
+{
+    Hash hash(htSHA1);
+    for (size_t i = 0; i < hash.hashSize; ++i)
+        hash.hash[i] = *gen::arbitrary<uint8_t>();
+    return gen::just(hash);
+}
+
+}
diff --git a/src/libutil/tests/hash.hh b/tests/unit/libutil-support/tests/hash.hh
similarity index 100%
rename from src/libutil/tests/hash.hh
rename to tests/unit/libutil-support/tests/hash.hh
diff --git a/src/libutil/tests/args.cc b/tests/unit/libutil/args.cc
similarity index 98%
rename from src/libutil/tests/args.cc
rename to tests/unit/libutil/args.cc
index bea74a8c8..950224430 100644
--- a/src/libutil/tests/args.cc
+++ b/tests/unit/libutil/args.cc
@@ -1,5 +1,5 @@
-#include "../args.hh"
-#include "libutil/fs-sink.hh"
+#include "args.hh"
+#include "fs-sink.hh"
 #include <list>
 
 #include <gtest/gtest.h>
@@ -165,4 +165,4 @@ RC_GTEST_PROP(
 
 #endif
 
-}
\ No newline at end of file
+}
diff --git a/src/libutil/tests/canon-path.cc b/tests/unit/libutil/canon-path.cc
similarity index 100%
rename from src/libutil/tests/canon-path.cc
rename to tests/unit/libutil/canon-path.cc
diff --git a/src/libutil/tests/chunked-vector.cc b/tests/unit/libutil/chunked-vector.cc
similarity index 100%
rename from src/libutil/tests/chunked-vector.cc
rename to tests/unit/libutil/chunked-vector.cc
diff --git a/src/libutil/tests/closure.cc b/tests/unit/libutil/closure.cc
similarity index 100%
rename from src/libutil/tests/closure.cc
rename to tests/unit/libutil/closure.cc
diff --git a/src/libutil/tests/compression.cc b/tests/unit/libutil/compression.cc
similarity index 100%
rename from src/libutil/tests/compression.cc
rename to tests/unit/libutil/compression.cc
diff --git a/src/libutil/tests/config.cc b/tests/unit/libutil/config.cc
similarity index 100%
rename from src/libutil/tests/config.cc
rename to tests/unit/libutil/config.cc
diff --git a/unit-test-data/libutil/git/check-data.sh b/tests/unit/libutil/data/git/check-data.sh
similarity index 98%
rename from unit-test-data/libutil/git/check-data.sh
rename to tests/unit/libutil/data/git/check-data.sh
index 68b705c95..b3f59c4f1 100644
--- a/unit-test-data/libutil/git/check-data.sh
+++ b/tests/unit/libutil/data/git/check-data.sh
@@ -2,7 +2,7 @@
 
 set -eu -o pipefail
 
-export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/git-hashing/unit-test-data
+export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/git-hashing/check-data
 mkdir -p $TEST_ROOT
 
 repo="$TEST_ROOT/scratch"
diff --git a/unit-test-data/libutil/git/hello-world-blob.bin b/tests/unit/libutil/data/git/hello-world-blob.bin
similarity index 100%
rename from unit-test-data/libutil/git/hello-world-blob.bin
rename to tests/unit/libutil/data/git/hello-world-blob.bin
diff --git a/unit-test-data/libutil/git/hello-world.bin b/tests/unit/libutil/data/git/hello-world.bin
similarity index 100%
rename from unit-test-data/libutil/git/hello-world.bin
rename to tests/unit/libutil/data/git/hello-world.bin
diff --git a/unit-test-data/libutil/git/tree.bin b/tests/unit/libutil/data/git/tree.bin
similarity index 100%
rename from unit-test-data/libutil/git/tree.bin
rename to tests/unit/libutil/data/git/tree.bin
diff --git a/unit-test-data/libutil/git/tree.txt b/tests/unit/libutil/data/git/tree.txt
similarity index 100%
rename from unit-test-data/libutil/git/tree.txt
rename to tests/unit/libutil/data/git/tree.txt
diff --git a/src/libutil/tests/git.cc b/tests/unit/libutil/git.cc
similarity index 97%
rename from src/libutil/tests/git.cc
rename to tests/unit/libutil/git.cc
index 2842ea4d0..551a2d105 100644
--- a/src/libutil/tests/git.cc
+++ b/tests/unit/libutil/git.cc
@@ -11,7 +11,7 @@ using namespace git;
 
 class GitTest : public CharacterizationTest
 {
-    Path unitTestData = getUnitTestData() + "/libutil/git";
+    Path unitTestData = getUnitTestData() + "/git";
 
 public:
 
@@ -86,8 +86,8 @@ TEST_F(GitTest, blob_write) {
 
 /**
  * This data is for "shallow" tree tests. However, we use "real" hashes
- * so that we can check our test data in the corresponding functional
- * test (`git-hashing/unit-test-data`).
+ * so that we can check our test data in a small shell script test test
+ * (`tests/unit/libutil/data/git/check-data.sh`).
  */
 const static Tree tree = {
     {
diff --git a/src/libutil/tests/hash.cc b/tests/unit/libutil/hash.cc
similarity index 92%
rename from src/libutil/tests/hash.cc
rename to tests/unit/libutil/hash.cc
index 9a5ebbb30..92291afce 100644
--- a/src/libutil/tests/hash.cc
+++ b/tests/unit/libutil/hash.cc
@@ -1,12 +1,8 @@
 #include <regex>
 
-#include <nlohmann/json.hpp>
 #include <gtest/gtest.h>
-#include <rapidcheck/gtest.h>
 
-#include <hash.hh>
-
-#include "tests/hash.hh"
+#include "hash.hh"
 
 namespace nix {
 
@@ -68,7 +64,6 @@ namespace nix {
                 "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd"
                 "454d4423643ce80e2a9ac94fa54ca49f");
     }
-
     TEST(hashString, testKnownSHA512Hashes2) {
         // values taken from: https://tools.ietf.org/html/rfc4634
         auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu";
@@ -95,16 +90,3 @@ namespace nix {
         ASSERT_EQ(parseHashFormatOpt("sha0042"), std::nullopt);
     }
 }
-
-namespace rc {
-using namespace nix;
-
-Gen<Hash> Arbitrary<Hash>::arbitrary()
-{
-    Hash hash(htSHA1);
-    for (size_t i = 0; i < hash.hashSize; ++i)
-        hash.hash[i] = *gen::arbitrary<uint8_t>();
-    return gen::just(hash);
-}
-
-}
diff --git a/src/libutil/tests/hilite.cc b/tests/unit/libutil/hilite.cc
similarity index 100%
rename from src/libutil/tests/hilite.cc
rename to tests/unit/libutil/hilite.cc
diff --git a/src/libutil/tests/json-utils.cc b/tests/unit/libutil/json-utils.cc
similarity index 100%
rename from src/libutil/tests/json-utils.cc
rename to tests/unit/libutil/json-utils.cc
diff --git a/tests/unit/libutil/local.mk b/tests/unit/libutil/local.mk
new file mode 100644
index 000000000..930efb90b
--- /dev/null
+++ b/tests/unit/libutil/local.mk
@@ -0,0 +1,31 @@
+check: libutil-tests_RUN
+
+programs += libutil-tests
+
+libutil-tests_NAME = libnixutil-tests
+
+libutil-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data
+
+libutil-tests_DIR := $(d)
+
+ifeq ($(INSTALL_UNIT_TESTS), yes)
+  libutil-tests_INSTALL_DIR := $(checkbindir)
+else
+  libutil-tests_INSTALL_DIR :=
+endif
+
+libutil-tests_SOURCES := $(wildcard $(d)/*.cc)
+
+libutil-tests_EXTRA_INCLUDES = \
+    -I tests/unit/libutil-support \
+    -I src/libutil
+
+libutil-tests_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES)
+
+libutil-tests_LIBS = libutil-test-support libutil
+
+libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS)
+
+check: $(d)/data/git/check-data.sh.test
+
+$(eval $(call run-test,$(d)/data/git/check-data.sh))
diff --git a/src/libutil/tests/logging.cc b/tests/unit/libutil/logging.cc
similarity index 100%
rename from src/libutil/tests/logging.cc
rename to tests/unit/libutil/logging.cc
diff --git a/src/libutil/tests/lru-cache.cc b/tests/unit/libutil/lru-cache.cc
similarity index 100%
rename from src/libutil/tests/lru-cache.cc
rename to tests/unit/libutil/lru-cache.cc
diff --git a/src/libutil/tests/pool.cc b/tests/unit/libutil/pool.cc
similarity index 100%
rename from src/libutil/tests/pool.cc
rename to tests/unit/libutil/pool.cc
diff --git a/src/libutil/tests/references.cc b/tests/unit/libutil/references.cc
similarity index 100%
rename from src/libutil/tests/references.cc
rename to tests/unit/libutil/references.cc
diff --git a/src/libutil/tests/suggestions.cc b/tests/unit/libutil/suggestions.cc
similarity index 100%
rename from src/libutil/tests/suggestions.cc
rename to tests/unit/libutil/suggestions.cc
diff --git a/src/libutil/tests/tests.cc b/tests/unit/libutil/tests.cc
similarity index 100%
rename from src/libutil/tests/tests.cc
rename to tests/unit/libutil/tests.cc
diff --git a/src/libutil/tests/url.cc b/tests/unit/libutil/url.cc
similarity index 100%
rename from src/libutil/tests/url.cc
rename to tests/unit/libutil/url.cc
diff --git a/src/libutil/tests/xml-writer.cc b/tests/unit/libutil/xml-writer.cc
similarity index 100%
rename from src/libutil/tests/xml-writer.cc
rename to tests/unit/libutil/xml-writer.cc

From 7355a48b1a4ce2e393598c2a72ef520cba9d172d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 1 Dec 2023 15:55:18 -0500
Subject: [PATCH 129/421] flake.lock: Update Nixpkgs to fix static build
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The problem was since switching to use libgit2, we had a package in our
closure (`http-parser`) that was always trying to build as a shared
object.

Underlying Nixpkgs PR (a 23.05 backport)
https://github.com/NixOS/nixpkgs/pull/271202

Flake lock file updates:

• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/9ba29e2346bc542e9909d1021e8fd7d4b3f64db0' (2023-11-13)
  → 'github:NixOS/nixpkgs/36c4ac09e9bebcec1fa7b7539cddb0c9e837409c' (2023-11-30)
---
 flake.lock | 8 ++++----
 flake.nix  | 8 +++++++-
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/flake.lock b/flake.lock
index f120d3b5f..3cb9e72c9 100644
--- a/flake.lock
+++ b/flake.lock
@@ -50,16 +50,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1700748986,
-        "narHash": "sha256-/nqLrNU297h3PCw4QyDpZKZEUHmialJdZW2ceYFobds=",
+        "lastModified": 1701355166,
+        "narHash": "sha256-4V7XMI0Gd+y0zsi++cEHd99u3GNL0xSTGRmiWKzGnUQ=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "9ba29e2346bc542e9909d1021e8fd7d4b3f64db0",
+        "rev": "36c4ac09e9bebcec1fa7b7539cddb0c9e837409c",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.05-small",
+        "ref": "staging-23.05",
         "repo": "nixpkgs",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index e2e510cbc..dbd45f053 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,7 +1,13 @@
 {
   description = "The purely functional package manager";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
+  # TODO Go back to nixos-23.05-small once
+  # https://github.com/NixOS/nixpkgs/pull/271202 is merged.
+  #
+  # Also, do not grab arbitrary further staging commits. This PR was
+  # carefully made to be based on release-23.05 and just contain
+  # rebuild-causing changes to packages that Nix actually uses.
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/staging-23.05";
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };

From 59c4c82aebb814d864548c3ad2e9128ab6e902bf Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Sat, 2 Dec 2023 00:56:23 +0100
Subject: [PATCH 130/421] fix links in stores overview

---
 doc/manual/generate-manpage.nix        | 3 ++-
 doc/manual/src/store/types/index.md.in | 8 ++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix
index 365422af7..ae31b2a1f 100644
--- a/doc/manual/generate-manpage.nix
+++ b/doc/manual/generate-manpage.nix
@@ -103,7 +103,8 @@ let
             ${allStores}
           '';
           index = replaceStrings
-            [ "@store-types@" ] [ storesOverview ]
+            [ "@store-types@" "./local-store.md" "./local-daemon-store.md" ]
+            [ storesOverview "#local-store" "#local-daemon-store" ]
             details.doc;
           storesOverview =
             let
diff --git a/doc/manual/src/store/types/index.md.in b/doc/manual/src/store/types/index.md.in
index b4db553a2..a35161ce8 100644
--- a/doc/manual/src/store/types/index.md.in
+++ b/doc/manual/src/store/types/index.md.in
@@ -29,15 +29,15 @@ supported settings for each store type are documented below.
 The special store URL `auto` causes Nix to automatically select a
 store as follows:
 
-* Use the [local store](#local-store) `/nix/store` if `/nix/var/nix`
+* Use the [local store](./local-store.md) `/nix/store` if `/nix/var/nix`
   is writable by the current user.
 
 * Otherwise, if `/nix/var/nix/daemon-socket/socket` exists, [connect
-  to the Nix daemon listening on that socket](#local-daemon-store).
+  to the Nix daemon listening on that socket](./local-daemon-store.md).
 
-* Otherwise, on Linux only, use the [local chroot store](#local-store)
+* Otherwise, on Linux only, use the [local chroot store](./local-store.md)
   `~/.local/share/nix/root`, which will be created automatically if it
   does not exist.
 
-* Otherwise, use the [local store](#local-store) `/nix/store`.
+* Otherwise, use the [local store](./local-store.md) `/nix/store`.
 

From 51adfb9b277fe54ad03fa2c9981585f123fcc200 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Sat, 2 Dec 2023 02:21:17 +0100
Subject: [PATCH 131/421] reword documentation on settings and attributes
 related to substitution

- add links
- be more concise
- clarify the distinction between `preferLocalBuild` and `allowSubstitutes`
---
 .../src/language/advanced-attributes.md       | 23 +++++--------------
 src/libstore/globals.hh                       |  4 +---
 2 files changed, 7 insertions(+), 20 deletions(-)

diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md
index 282b75af2..5a6c00cd4 100644
--- a/doc/manual/src/language/advanced-attributes.md
+++ b/doc/manual/src/language/advanced-attributes.md
@@ -257,29 +257,18 @@ Derivations can declare some infrequently used optional attributes.
     of the environment (typically, a few hundred kilobyte).
 
   - [`preferLocalBuild`]{#adv-attr-preferLocalBuild}\
-    If this attribute is set to `true` and [distributed building is
-    enabled](../advanced-topics/distributed-builds.md), then, if
-    possible, the derivation will be built locally instead of forwarded
-    to a remote machine. This is appropriate for trivial builders
-    where the cost of doing a download or remote build would exceed
-    the cost of building locally.
+    If this attribute is set to `true` and [distributed building is enabled](../advanced-topics/distributed-builds.md), then, if possible, the derivation will be built locally instead of being forwarded to a remote machine.
+    This is useful for derivations that are cheapest to build locally.
 
   - [`allowSubstitutes`]{#adv-attr-allowSubstitutes}\
-    If this attribute is set to `false`, then Nix will always build this
-    derivation; it will not try to substitute its outputs. This is
-    useful for very trivial derivations (such as `writeText` in Nixpkgs)
-    that are cheaper to build than to substitute from a binary cache.
+    If this attribute is set to `false`, then Nix will always build this derivation (locally or remotely); it will not try to substitute its outputs.
+    This is useful for derivations that are cheaper to build than to substitute.
 
-    You may disable the effects of this attibute by enabling the
-    `always-allow-substitutes` configuration option in Nix.
+    This attribute can be ignored by setting [`always-allow-substitutes`](@docroot@/command-ref/conf-file.md#conf-always-allow-substitutes) to `true`.
 
     > **Note**
     >
-    > You need to have a builder configured which satisfies the
-    > derivation’s `system` attribute, since the derivation cannot be
-    > substituted. Thus it is usually a good idea to align `system` with
-    > `builtins.currentSystem` when setting `allowSubstitutes` to
-    > `false`. For most trivial derivations this should be the case.
+    > If set to `false`, the [`builder`](./derivations.md#attr-builder) should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted.
 
   - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\
     If the special attribute `__structuredAttrs` is set to `true`, the other derivation
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 38b0d516c..36ba51e23 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -268,9 +268,7 @@ public:
     Setting<bool> alwaysAllowSubstitutes{
         this, false, "always-allow-substitutes",
         R"(
-          If set to `true`, Nix will ignore the `allowSubstitutes` attribute in
-          derivations and always attempt to use available substituters.
-          For more information on `allowSubstitutes`, see [the manual chapter on advanced attributes](../language/advanced-attributes.md).
+          If set to `true`, Nix will ignore the [`allowSubstitutes`](@docroot@/language/advanced-attributes.md) attribute in derivations and always attempt to use [available substituters](#conf-substituters).
         )"};
 
     Setting<bool> buildersUseSubstitutes{

From 24b781773f3d24b62b1c36154c36fc98417cbcdb Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Sat, 2 Dec 2023 03:02:58 +0100
Subject: [PATCH 132/421] fix random docs errors

remove link to the contributing guide from user documentation.
it doesn't help here, and the target at first glance shows redundant
information.
---
 src/libexpr/primops.cc  | 2 +-
 src/libstore/globals.hh | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 63c90795a..c2499bdae 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -4400,7 +4400,7 @@ void EvalState::createBaseEnv()
     addConstant("__currentSystem", v, {
         .type = nString,
         .doc = R"(
-          The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-pure-eval).
+          The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-system).
 
           It can be used to set the `system` attribute for [`builtins.derivation`](@docroot@/language/derivations.md) such that the resulting derivation can be built on the same system that evaluates the Nix expression:
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 38b0d516c..8da9e371f 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -201,7 +201,7 @@ public:
           Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in [`extra-platforms`](#conf-extra-platforms).
 
           The default value is set when Nix itself is compiled for the system it will run on.
-          The following system types are widely used, as [Nix is actively supported on these platforms](@docroot@/contributing/hacking.md#platforms):
+          The following system types are widely used, as Nix is actively supported on these platforms:
 
           - `x86_64-linux`
           - `x86_64-darwin`
@@ -761,7 +761,7 @@ public:
         "substituters",
         R"(
           A list of [URLs of Nix stores](@docroot@/store/types/index.md#store-url-format) to be used as substituters, separated by whitespace.
-          A substituter is an additional [store]{@docroot@/glossary.md##gloss-store} from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them.
+          A substituter is an additional [store](@docroot@/glossary.md#gloss-store) from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them.
 
           Substituters are tried based on their priority value, which each substituter can set independently.
           Lower value means higher priority.

From 368fdb482da039fd40a1a51bbf851c54f65eb4c5 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Sat, 2 Dec 2023 03:06:47 +0100
Subject: [PATCH 133/421] reword description of the `builders-use-substitutes`
 setting

---
 src/libstore/globals.hh | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 38b0d516c..dcdcd31d5 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -276,13 +276,10 @@ public:
     Setting<bool> buildersUseSubstitutes{
         this, false, "builders-use-substitutes",
         R"(
-          If set to `true`, Nix will instruct remote build machines to use
-          their own binary substitutes if available. In practical terms, this
-          means that remote hosts will fetch as many build dependencies as
-          possible from their own substitutes (e.g, from `cache.nixos.org`),
-          instead of waiting for this host to upload them all. This can
-          drastically reduce build times if the network connection between
-          this computer and the remote build host is slow.
+          If set to `true`, Nix will instruct [remote build machines](#conf-builders) to use their own [`substituters`](#conf-substituters) if available.
+
+          It means that remote build hosts will fetch as many dependencies as possible from their own substituters (e.g, from `cache.nixos.org`) instead of waiting for the local machine to upload them all.
+          This can drastically reduce build times if the network connection between the local machine and the remote build host is slow.
         )"};
 
     Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space",

From 5b281ddf50775ff37577f80cd3f1f7dbf76c9762 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Sat, 2 Dec 2023 02:13:11 +0100
Subject: [PATCH 134/421] reword description of the `max-jobs` setting

- remove prose for the default value, which is shown programmatically
- add note on how this relates to `cores`
- add link to mentioned derivation attribute
---
 src/libstore/globals.hh | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 38b0d516c..7a30c5ae2 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -151,13 +151,18 @@ public:
     MaxBuildJobsSetting maxBuildJobs{
         this, 1, "max-jobs",
         R"(
-          This option defines the maximum number of jobs that Nix will try to
-          build in parallel. The default is `1`. The special value `auto`
-          causes Nix to use the number of CPUs in your system. `0` is useful
-          when using remote builders to prevent any local builds (except for
-          `preferLocalBuild` derivation attribute which executes locally
-          regardless). It can be overridden using the `--max-jobs` (`-j`)
-          command line switch.
+          Maximum number of jobs that Nix will try to build locally in parallel.
+
+          The special value `auto` causes Nix to use the number of CPUs in your system.
+          Use `0` to disable local builds and directly use the remote machines specified in [`builders`](#conf-builders).
+          This will not affect derivations that have [`preferLocalBuild = true`](@docroot@/language/advanced-attributes.md#adv-attr-preferLocalBuild), which are always built locally.
+
+          > **Note**
+          >
+          > The number of CPU cores to use for each build job is independently determined by the [`cores`](#conf-cores) setting.
+
+          <!-- TODO(@fricklerhandwerk): would be good to have those shorthands for common options as part of the specification -->
+          The setting can be overridden using the `--max-jobs` (`-j`) command line switch.
         )",
         {"build-max-jobs"}};
 

From 2c3749a335d4462412ac73eb77a81d949e1e8ba6 Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Sat, 2 Dec 2023 16:08:06 +0000
Subject: [PATCH 135/421] Fix cross builds

---
 package.nix | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/package.nix b/package.nix
index bed77ba3b..9f30eef2f 100644
--- a/package.nix
+++ b/package.nix
@@ -123,6 +123,10 @@ stdenv.mkDerivation (finalAttrs: {
     openssl
     sqlite
     xz
+
+    # These could be checkInputs but the configure phase fails w/o them
+    gtest
+    rapidcheck
   ]
   ++ lib.optional stdenv.isLinux libseccomp
   ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid
@@ -137,14 +141,13 @@ stdenv.mkDerivation (finalAttrs: {
   doCheck = true;
 
   checkInputs = [
-    gtest
-    rapidcheck
+    # see buildInputs. The configure script always wants its test libs
   ];
 
   nativeCheckInputs = [
     git
-    mercurial # FIXME: remove? only needed for tests
-    openssh # only needed for tests (ssh-keygen)
+    mercurial
+    openssh
   ];
 
   propagatedBuildInputs = [

From ca598328085fe7a379bff8777031101fba80921b Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Sat, 2 Dec 2023 16:36:59 +0000
Subject: [PATCH 136/421] Fix coverage.nix

---
 coverage.nix | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/coverage.nix b/coverage.nix
index 2390ef52d..f952d8b09 100644
--- a/coverage.nix
+++ b/coverage.nix
@@ -14,19 +14,21 @@ releaseTools.coverageAnalysis {
 
   inherit (nix)
     src
-    configureFlags
-    nativeBuildInputs
     buildInputs
-    #checkInputs
+    nativeBuildInputs
+    propagatedBuildInputs
+    configureFlags
+    makeFlags
+    installFlags
+    doInstallCheck
+    installCheckFlags
+    installCheckTarget
     ;
 
   enableParallelBuilding = true;
 
   dontInstall = false;
 
-  doInstallCheck = true;
-  installCheckTarget = "installcheck"; # work around buggy detection in stdenv
-
   lcovFilter = [ "*/boost/*" "*-tab.*" ];
 
   hardeningDisable = ["fortify"];

From 118fa9689ab0e6d12b360708177f9a1b56f3d466 Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Sat, 2 Dec 2023 16:42:01 +0000
Subject: [PATCH 137/421] Create internal-api-docs.nix

---
 flake.nix             | 17 +----------------
 internal-api-docs.nix | 24 ++++++++++++++++++++++++
 2 files changed, 25 insertions(+), 16 deletions(-)
 create mode 100644 internal-api-docs.nix

diff --git a/flake.nix b/flake.nix
index c0841a76d..b1c3a777e 100644
--- a/flake.nix
+++ b/flake.nix
@@ -482,22 +482,7 @@
         coverage = nixpkgsFor.x86_64-linux.native.callPackage ./coverage.nix {};
 
         # API docs for Nix's unstable internal C++ interfaces.
-        internal-api-docs = nixpkgsFor.x86_64-linux.native.nix.overrideAttrs (old: {
-          pname = "nix-internal-api-docs";
-
-          configureFlags = old.configureFlags ++ [ "--enable-internal-api-docs" ];
-          nativeBuildInputs = old.nativeBuildInputs ++ [ nixpkgsFor.x86_64-linux.native.doxygen ];
-
-          dontBuild = true;
-          doCheck = false;
-
-          installTargets = [ "internal-api-html" ];
-
-          postInstall = ''
-            mkdir -p $out/nix-support
-            echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products
-          '';
-        });
+        internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./internal-api-docs.nix {};
 
         # System tests.
         tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // {
diff --git a/internal-api-docs.nix b/internal-api-docs.nix
new file mode 100644
index 000000000..ddd3fa891
--- /dev/null
+++ b/internal-api-docs.nix
@@ -0,0 +1,24 @@
+{ nix
+, doxygen
+}:
+
+nix.overrideAttrs (old: {
+  pname = "nix-internal-api-docs";
+
+  configureFlags = old.configureFlags ++ [
+    "--enable-internal-api-docs"
+  ];
+  nativeBuildInputs = old.nativeBuildInputs ++ [
+    doxygen
+  ];
+
+  dontBuild = true;
+  doCheck = false;
+
+  installTargets = [ "internal-api-html" ];
+
+  postInstall = ''
+    mkdir -p $out/nix-support
+    echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products
+  '';
+})

From 19d41fb20a45d2bf66f78813514bf5c5fd420a8b Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Sat, 2 Dec 2023 17:25:47 +0000
Subject: [PATCH 138/421] Fix stuff

---
 flake.nix             | 230 +++---------------------------------------
 package.nix           |  60 ++++++-----
 test-nix-versions.nix |  50 +++++++++
 3 files changed, 96 insertions(+), 244 deletions(-)
 create mode 100644 test-nix-versions.nix

diff --git a/flake.nix b/flake.nix
index b1c3a777e..fbce13604 100644
--- a/flake.nix
+++ b/flake.nix
@@ -12,6 +12,14 @@
     let
       inherit (nixpkgs) lib;
 
+      # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981
+      # Not an "idiomatic" flake input because:
+      #  - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730
+      #  - Subflake would download redundant and huge parent flake
+      #  - No git tree hash support: https://github.com/NixOS/nix/issues/6044
+      inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; }))
+        fileset;
+
       officialRelease = false;
 
       # Set to true to build the release notes for the next release.
@@ -56,57 +64,6 @@
             })
             stdenvs);
 
-      # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981
-      # Not an "idiomatic" flake input because:
-      #  - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730
-      #  - Subflake would download redundant and huge parent flake
-      #  - No git tree hash support: https://github.com/NixOS/nix/issues/6044
-      inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; }))
-        fileset;
-
-      baseFiles =
-        # .gitignore has already been processed, so any changes in it are irrelevant
-        # at this point. It is not represented verbatim for test purposes because
-        # that would interfere with repo semantics.
-        fileset.fileFilter (f: f.name != ".gitignore") ./.;
-
-      configureFiles = fileset.unions [
-        ./.version
-        ./configure.ac
-        ./m4
-        # TODO: do we really need README.md? It doesn't seem used in the build.
-        ./README.md
-      ];
-
-      topLevelBuildFiles = fileset.unions [
-        ./local.mk
-        ./Makefile
-        ./Makefile.config.in
-        ./mk
-      ];
-
-      functionalTestFiles = fileset.unions [
-        ./tests/functional
-        (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
-      ];
-
-      nixSrc = fileset.toSource {
-        root = ./.;
-        fileset = fileset.intersect baseFiles (fileset.unions [
-          configureFiles
-          topLevelBuildFiles
-          ./boehmgc-coroutine-sp-fallback.diff
-          ./doc
-          ./misc
-          ./precompiled-headers.h
-          ./src
-          ./unit-test-data
-          ./COPYING
-          ./scripts/local.mk
-          functionalTestFiles
-        ]);
-      };
-
       # Memoize nixpkgs for different platforms for efficiency.
       nixpkgsFor = forAllSystems
         (system: let
@@ -131,130 +88,6 @@
           cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv");
         });
 
-      commonDeps =
-        { pkgs
-        , isStatic ? pkgs.stdenv.hostPlatform.isStatic
-        }:
-        with pkgs; rec {
-        # Use "busybox-sandbox-shell" if present,
-        # if not (legacy) fallback and hope it's sufficient.
-        sh = pkgs.busybox-sandbox-shell or (busybox.override {
-          useMusl = true;
-          enableStatic = true;
-          enableMinimal = true;
-          extraConfig = ''
-            CONFIG_FEATURE_FANCY_ECHO y
-            CONFIG_FEATURE_SH_MATH y
-            CONFIG_FEATURE_SH_MATH_64 y
-
-            CONFIG_ASH y
-            CONFIG_ASH_OPTIMIZE_FOR_SIZE y
-
-            CONFIG_ASH_ALIAS y
-            CONFIG_ASH_BASH_COMPAT y
-            CONFIG_ASH_CMDCMD y
-            CONFIG_ASH_ECHO y
-            CONFIG_ASH_GETOPTS y
-            CONFIG_ASH_INTERNAL_GLOB y
-            CONFIG_ASH_JOB_CONTROL y
-            CONFIG_ASH_PRINTF y
-            CONFIG_ASH_TEST y
-          '';
-        });
-
-        configureFlags =
-          lib.optionals stdenv.isLinux [
-            "--with-boost=${boost-nix}/lib"
-            "--with-sandbox-shell=${sh}/bin/busybox"
-          ]
-          ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [
-            "LDFLAGS=-fuse-ld=gold"
-          ];
-
-        testConfigureFlags = [
-          "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include"
-        ] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
-          "--enable-install-unit-tests"
-          "--with-check-bin-dir=${builtins.placeholder "check"}/bin"
-          "--with-check-lib-dir=${builtins.placeholder "check"}/lib"
-        ];
-
-        internalApiDocsConfigureFlags = [
-          "--enable-internal-api-docs"
-        ];
-
-        inherit (pkgs.buildPackages) changelog-d;
-
-        nativeBuildDeps =
-          [
-            buildPackages.bison
-            buildPackages.flex
-            (lib.getBin buildPackages.lowdown-nix)
-            buildPackages.mdbook
-            buildPackages.mdbook-linkcheck
-            buildPackages.autoconf-archive
-            buildPackages.autoreconfHook
-            buildPackages.pkg-config
-
-            # Tests
-            buildPackages.git
-            buildPackages.mercurial # FIXME: remove? only needed for tests
-            buildPackages.jq # Also for custom mdBook preprocessor.
-            buildPackages.openssh # only needed for tests (ssh-keygen)
-          ]
-          ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]
-          # Official releases don't have rl-next, so we don't need to compile a changelog
-          ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d
-          ;
-
-        buildDeps =
-          [ curl
-            bzip2 xz brotli editline
-            openssl sqlite
-            libarchive
-            (pkgs.libgit2.overrideAttrs (attrs: {
-              src = libgit2;
-              version = libgit2.lastModifiedDate;
-              cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"];
-            }))
-            boost-nix
-            lowdown-nix
-            libsodium
-          ]
-          ++ lib.optionals stdenv.isLinux [libseccomp]
-          ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid;
-
-        checkDeps = [
-          gtest
-          rapidcheck
-        ];
-
-        internalApiDocsDeps = [
-          buildPackages.doxygen
-        ];
-
-        awsDeps = lib.optional (stdenv.isLinux || stdenv.isDarwin)
-          (aws-sdk-cpp.override {
-            apis = ["s3" "transfer"];
-            customMemoryManagement = false;
-          });
-
-        propagatedDeps =
-          [ ((boehmgc.override {
-              enableLargeConfig = true;
-            }).overrideAttrs(o: {
-              patches = (o.patches or []) ++ [
-                ./boehmgc-coroutine-sp-fallback.diff
-
-                # https://github.com/ivmai/bdwgc/pull/586
-                ./boehmgc-traceable_allocator-public.diff
-              ];
-            })
-            )
-            nlohmann_json
-          ];
-      };
-
       installScriptFor = systems:
         with nixpkgsFor.x86_64-linux.native;
         runCommand "installer-script"
@@ -289,50 +122,11 @@
             echo "file installer $out/install" >> $out/nix-support/hydra-build-products
           '';
 
-      testNixVersions = pkgs: client: daemon: with commonDeps { inherit pkgs; }; with pkgs.lib; pkgs.stdenv.mkDerivation {
-        NIX_DAEMON_PACKAGE = daemon;
-        NIX_CLIENT_PACKAGE = client;
-        name =
-          "nix-tests"
-          + optionalString
-            (versionAtLeast daemon.version "2.4pre20211005" &&
-             versionAtLeast client.version "2.4pre20211005")
-            "-${client.version}-against-${daemon.version}";
-        inherit version;
-
-        src = fileset.toSource {
-          root = ./.;
-          fileset = fileset.intersect baseFiles (fileset.unions [
-            configureFiles
-            topLevelBuildFiles
-            functionalTestFiles
-          ]);
+      testNixVersions = pkgs: client: daemon:
+        pkgs.callPackage ./test-nix-versions.nix {
+          inherit client daemon fileset;
         };
 
-        VERSION_SUFFIX = versionSuffix;
-
-        nativeBuildInputs = nativeBuildDeps;
-        buildInputs = buildDeps ++ awsDeps ++ checkDeps;
-        propagatedBuildInputs = propagatedDeps;
-
-        enableParallelBuilding = true;
-
-        configureFlags =
-          testConfigureFlags # otherwise configure fails
-          ++ [ "--disable-build" ];
-        dontBuild = true;
-        doInstallCheck = true;
-
-        installPhase = ''
-          mkdir -p $out
-        '';
-
-        installCheckPhase = ''
-          mkdir -p src/nix-channel
-          make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES
-        '';
-      };
-
       binaryTarball = nix: pkgs: pkgs.callPackage ./binary-tarball.nix {
         inherit nix;
       };
@@ -491,7 +285,7 @@
           # on a particular version of Nixpkgs.
           evalNixpkgs =
             let
-              inherit (nixpkgsFor.x86_64-linux.native) runCommand nix nixpkgs-regression;
+              inherit (nixpkgsFor.x86_64-linux.native) runCommand nix;
             in
             runCommand "eval-nixos" { buildInputs = [ nix ]; }
               ''
diff --git a/package.nix b/package.nix
index 9f30eef2f..e4c66958b 100644
--- a/package.nix
+++ b/package.nix
@@ -43,6 +43,30 @@
 let
   version = lib.fileContents ./.version + versionSuffix;
   canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
+
+  filesets = {
+    baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.;
+
+    configureFiles = fileset.unions [
+      ./.version
+      ./configure.ac
+      ./m4
+      # TODO: do we really need README.md? It doesn't seem used in the build.
+      ./README.md
+    ];
+
+    topLevelBuildFiles = fileset.unions [
+      ./local.mk
+      ./Makefile
+      ./Makefile.config.in
+      ./mk
+    ];
+
+    functionalTestFiles = fileset.unions [
+      ./tests/functional
+      (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
+    ];
+  };
 in
 
 stdenv.mkDerivation (finalAttrs: {
@@ -51,33 +75,13 @@ stdenv.mkDerivation (finalAttrs: {
 
   src =
     let
-      baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.;
-      configureFiles = fileset.unions [
-        ./.version
-        ./configure.ac
-        ./m4
-        # TODO: do we really need README.md? It doesn't seem used in the build.
-        ./README.md
-      ];
-
-      topLevelBuildFiles = fileset.unions [
-        ./local.mk
-        ./Makefile
-        ./Makefile.config.in
-        ./mk
-      ];
-
-      functionalTestFiles = fileset.unions [
-        ./tests/functional
-        (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
-      ];
 
     in
       fileset.toSource {
         root = ./.;
-        fileset = fileset.intersect baseFiles (fileset.unions [
-          configureFiles
-          topLevelBuildFiles
+        fileset = fileset.intersect filesets.baseFiles (fileset.unions [
+          filesets.configureFiles
+          filesets.topLevelBuildFiles
           ./boehmgc-coroutine-sp-fallback.diff
           ./doc
           ./misc
@@ -86,7 +90,7 @@ stdenv.mkDerivation (finalAttrs: {
           ./unit-test-data
           ./COPYING
           ./scripts/local.mk
-          functionalTestFiles
+          filesets.functionalTestFiles
         ]);
       };
 
@@ -231,8 +235,12 @@ stdenv.mkDerivation (finalAttrs: {
 
   hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
 
-  passthru.perl-bindings = callPackage ./perl {
-    inherit fileset stdenv;
+  passthru ={
+    inherit filesets;
+
+    perl-bindings = callPackage ./perl {
+      inherit fileset stdenv;
+    };
   };
 
   meta.platforms = lib.platforms.unix;
diff --git a/test-nix-versions.nix b/test-nix-versions.nix
new file mode 100644
index 000000000..15f6cd8d0
--- /dev/null
+++ b/test-nix-versions.nix
@@ -0,0 +1,50 @@
+{ lib
+, fileset
+, stdenv
+, client
+, daemon
+}:
+
+stdenv.mkDerivation {
+  NIX_DAEMON_PACKAGE = daemon;
+  NIX_CLIENT_PACKAGE = client;
+  name =
+    "nix-tests"
+    + lib.optionalString
+      (lib.versionAtLeast daemon.version "2.4pre20211005" &&
+       lib.versionAtLeast client.version "2.4pre20211005")
+      "-${client.version}-against-${daemon.version}";
+
+  inherit (client)
+    version
+    VERSION_SUFFIX
+    nativeBuildInputs
+    buildInputs
+    propagatedBuildInputs
+    ;
+
+  src = fileset.toSource {
+    root = ./.;
+    fileset = with client.passthru.filesets;
+      fileset.intersect baseFiles (fileset.unions [
+        configureFiles
+        topLevelBuildFiles
+        functionalTestFiles
+      ]);
+  };
+
+  configureFlags = client.configureFlags # otherwise configure fails
+    ++ [ "--disable-build" ];
+
+  dontBuild = true;
+  doInstallCheck = true;
+
+  installPhase = ''
+    mkdir -p $out
+  '';
+
+  installCheckPhase = ''
+    mkdir -p src/nix-channel
+    make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES
+  '';
+}

From 0ca49b0c8663ae82931780ae3f1f45115b966285 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 3 Dec 2023 12:47:07 -0500
Subject: [PATCH 139/421] Add installing unit test flags

---
 package.nix | 37 ++++++++++++++++++++++++++-----------
 1 file changed, 26 insertions(+), 11 deletions(-)

diff --git a/package.nix b/package.nix
index e4c66958b..c1a3b9455 100644
--- a/package.nix
+++ b/package.nix
@@ -38,6 +38,17 @@
 , sqlite
 , util-linux
 , xz
+
+# Configuration Options
+#
+# This probably seems like too many degrees of freedom, but it
+# faithfully reflects how the underlying configure + make build system
+# work. The top-level flake.nix will choose useful combinations.
+
+# Whether to install unit tests. This is useful when cross compiling
+# since we cannot run them natively during the build, but can do so
+# later.
+, installUnitTests ? stdenv.hostPlatform != stdenv.buildPlatform
 }:
 
 let
@@ -69,7 +80,13 @@ let
   };
 in
 
-stdenv.mkDerivation (finalAttrs: {
+stdenv.mkDerivation (finalAttrs: let
+
+  # Either running the unit tests during the build, or installing them
+  # to be run later, requiresthe unit tests to be built.
+  buildUnitTests = finalAttrs.doCheck || installUnitTests;
+
+in {
   pname = "nix";
   inherit version;
 
@@ -97,7 +114,7 @@ stdenv.mkDerivation (finalAttrs: {
   VERSION_SUFFIX = versionSuffix;
 
   outputs = [ "out" "dev" "doc" ]
-    ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "check";
+    ++ lib.optional installUnitTests "check";
 
   nativeBuildInputs = [
     bison
@@ -142,7 +159,7 @@ stdenv.mkDerivation (finalAttrs: {
     })
   ;
 
-  doCheck = true;
+  doCheck = stdenv.hostPlatform != stdenv.buildPlatform;
 
   checkInputs = [
     # see buildInputs. The configure script always wants its test libs
@@ -190,14 +207,12 @@ stdenv.mkDerivation (finalAttrs: {
       "LDFLAGS=-fuse-ld=gold"
     ++ [ "--sysconfdir=/etc" ]
     ++ lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell"
-    ++ [ (lib.enableFeature finalAttrs.doCheck "tests") ]
-    ++ lib.optionals finalAttrs.doCheck (
-      [ "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" ]
-      ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
-        "--enable-install-unit-tests"
-        "--with-check-bin-dir=${builtins.placeholder "check"}/bin"
-        "--with-check-lib-dir=${builtins.placeholder "check"}/lib"
-      ])
+    ++ lib.optional buildUnitTests "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include"
+    ++ lib.optionals installUnitTests [
+      "--enable-install-unit-tests"
+      "--with-check-bin-dir=${builtins.placeholder "check"}/bin"
+      "--with-check-lib-dir=${builtins.placeholder "check"}/lib"
+    ]
     ++ lib.optional (!canRunInstalled) "--disable-doc-gen";
 
   enableParallelBuilding = true;

From ce598bae144c49c61b33cdf55679ef597ede9485 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 3 Dec 2023 14:10:09 -0500
Subject: [PATCH 140/421] WIP

---
 coverage.nix          |   8 ---
 flake.nix             |   7 ++-
 internal-api-docs.nix |  24 -------
 package.nix           | 143 ++++++++++++++++++++++++++++++++----------
 test-nix-versions.nix |  35 -----------
 5 files changed, 117 insertions(+), 100 deletions(-)
 delete mode 100644 internal-api-docs.nix

diff --git a/coverage.nix b/coverage.nix
index f952d8b09..2c5e4a06d 100644
--- a/coverage.nix
+++ b/coverage.nix
@@ -26,12 +26,4 @@ releaseTools.coverageAnalysis {
     ;
 
   enableParallelBuilding = true;
-
-  dontInstall = false;
-
-  lcovFilter = [ "*/boost/*" "*-tab.*" ];
-
-  hardeningDisable = ["fortify"];
-
-  NIX_CFLAGS_COMPILE = "-DCOVERAGE=1";
 }
diff --git a/flake.nix b/flake.nix
index fbce13604..44ce2d306 100644
--- a/flake.nix
+++ b/flake.nix
@@ -276,7 +276,12 @@
         coverage = nixpkgsFor.x86_64-linux.native.callPackage ./coverage.nix {};
 
         # API docs for Nix's unstable internal C++ interfaces.
-        internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./internal-api-docs.nix {};
+        internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix {
+          doBuild = false;
+          doCheck = false;
+          doInstallCheck = false;
+          enableInternalAPIDocs = true;
+        };
 
         # System tests.
         tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // {
diff --git a/internal-api-docs.nix b/internal-api-docs.nix
deleted file mode 100644
index ddd3fa891..000000000
--- a/internal-api-docs.nix
+++ /dev/null
@@ -1,24 +0,0 @@
-{ nix
-, doxygen
-}:
-
-nix.overrideAttrs (old: {
-  pname = "nix-internal-api-docs";
-
-  configureFlags = old.configureFlags ++ [
-    "--enable-internal-api-docs"
-  ];
-  nativeBuildInputs = old.nativeBuildInputs ++ [
-    doxygen
-  ];
-
-  dontBuild = true;
-  doCheck = false;
-
-  installTargets = [ "internal-api-html" ];
-
-  postInstall = ''
-    mkdir -p $out/nix-support
-    echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products
-  '';
-})
diff --git a/package.nix b/package.nix
index c1a3b9455..39fee8472 100644
--- a/package.nix
+++ b/package.nix
@@ -1,6 +1,7 @@
 { lib
 , callPackage
 , stdenv
+, releaseTools
 , versionSuffix ? ""
 , officialRelease ? false
 , buildUnreleasedNotes ? false
@@ -21,6 +22,7 @@
 , git
 , gtest
 , jq
+, doxygen
 , libarchive
 , libcpuid
 , libgit2
@@ -45,16 +47,35 @@
 # faithfully reflects how the underlying configure + make build system
 # work. The top-level flake.nix will choose useful combinations.
 
+, pname ? "nix"
+
+, doBuild ? true
+, doCheck ? stdenv.buildPlatform.canExecute stdenv.hostPlatform
+, doInstallCheck ? stdenv.buildPlatform.canExecute stdenv.hostPlatform
+
+, withCoverageChecks ? false
+
+# Whether to build the internal API docs, can be done separately from
+# everything else.
+, enableInternalAPIDocs ? false
+
 # Whether to install unit tests. This is useful when cross compiling
 # since we cannot run them natively during the build, but can do so
 # later.
 , installUnitTests ? stdenv.hostPlatform != stdenv.buildPlatform
+
+, test-daemon ? null
+, test-client ? null
 }:
 
 let
   version = lib.fileContents ./.version + versionSuffix;
   canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
 
+  attrs = {
+    inherit doBuild doCheck doInstallCheck;
+  };
+
   filesets = {
     baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.;
 
@@ -78,17 +99,30 @@ let
       (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
     ];
   };
+
+  mkDerivation =
+    if withCoverageChecks
+    then releaseTools.coverageAnalysis
+    else stdenv.mkDerivation;
 in
 
-stdenv.mkDerivation (finalAttrs: let
+mkDerivation (finalAttrs: let
+
+  inherit (finalAttrs)
+    doCheck
+    doInstallCheck
+    ;
+
+  doBuild = !finalAttrs.dontBuild;
 
   # Either running the unit tests during the build, or installing them
   # to be run later, requiresthe unit tests to be built.
-  buildUnitTests = finalAttrs.doCheck || installUnitTests;
+  buildUnitTests = doCheck || installUnitTests;
+
+  anySortOfTesting = buildUnitTests || doInstallCheck;
 
 in {
-  pname = "nix";
-  inherit version;
+  inherit pname version;
 
   src =
     let
@@ -96,9 +130,10 @@ in {
     in
       fileset.toSource {
         root = ./.;
-        fileset = fileset.intersect filesets.baseFiles (fileset.unions [
+        fileset = fileset.intersect filesets.baseFiles (fileset.unions ([
           filesets.configureFiles
           filesets.topLevelBuildFiles
+        ] ++ lib.optionals doBuild [
           ./boehmgc-coroutine-sp-fallback.diff
           ./doc
           ./misc
@@ -107,8 +142,9 @@ in {
           ./unit-test-data
           ./COPYING
           ./scripts/local.mk
+        ] ++ lib.optionals anySortOfTesting [
           filesets.functionalTestFiles
-        ]);
+        ]));
       };
 
   VERSION_SUFFIX = versionSuffix;
@@ -159,7 +195,13 @@ in {
     })
   ;
 
-  doCheck = stdenv.hostPlatform != stdenv.buildPlatform;
+  propagatedBuildInputs = [
+    boehmgc
+    nlohmann_json
+  ];
+
+  dontBuild = !attrs.doBuild;
+  doCheck = attrs.doCheck;
 
   checkInputs = [
     # see buildInputs. The configure script always wants its test libs
@@ -169,11 +211,8 @@ in {
     git
     mercurial
     openssh
-  ];
-
-  propagatedBuildInputs = [
-    boehmgc
-    nlohmann_json
+  ] ++ lib.optionals enableInternalAPIDocs [
+    doxygen
   ];
 
   disallowedReferences = [ boost ];
@@ -198,30 +237,41 @@ in {
     ''}
   '';
 
-  configureFlags =
-    lib.optionals stdenv.isLinux [
-      "--with-boost=${boost}/lib"
-      "--with-sandbox-shell=${sh}/bin/busybox"
-    ]
-    ++ lib.optional (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux"))
-      "LDFLAGS=-fuse-ld=gold"
-    ++ [ "--sysconfdir=/etc" ]
+  configureFlags = [
+    "--sysconfdir=/etc"
+    (lib.enableFeature doBuild "build")
+    (lib.enableFeature anySortOfTesting "test")
+    (lib.enableFeature enableInternalAPIDocs "internal-api-docs")
+    (lib.enableFeature canRunInstalled "doc-gen")
+    (lib.enableFeature installUnitTests "install-unit-tests")
+  ] ++ lib.optionals installUnitTests [
+    "--with-check-bin-dir=${builtins.placeholder "check"}/bin"
+    "--with-check-lib-dir=${builtins.placeholder "check"}/lib"
+  ] ++ lib.optionals stdenv.isLinux [
+    "--with-boost=${boost}/lib"
+    "--with-sandbox-shell=${sh}/bin/busybox"
+  ] ++ lib.optional (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux"))
+       "LDFLAGS=-fuse-ld=gold"
     ++ lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell"
-    ++ lib.optional buildUnitTests "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include"
-    ++ lib.optionals installUnitTests [
-      "--enable-install-unit-tests"
-      "--with-check-bin-dir=${builtins.placeholder "check"}/bin"
-      "--with-check-lib-dir=${builtins.placeholder "check"}/lib"
-    ]
-    ++ lib.optional (!canRunInstalled) "--disable-doc-gen";
+    ++ lib.optional buildUnitTests "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include";
 
   enableParallelBuilding = true;
 
   makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1";
 
+  installTargets = lib.optional doBuild "install"
+    ++ lib.optional enableInternalAPIDocs "internal-api-html";
+
   installFlags = "sysconfdir=$(out)/etc";
 
-  postInstall = ''
+  # In this case we are probably just running tests, and so there isn't
+  # anything to install, we just make an empty directory to signify tests
+  # succeeded.
+  installPhase = if finalAttrs.installTargets != [] then null else ''
+    mkdir -p $out
+  '';
+
+  postInstall = lib.optionalString doBuild ''
     mkdir -p $doc/nix-support
     echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
     ${lib.optionalString stdenv.hostPlatform.isStatic ''
@@ -238,19 +288,29 @@ in {
       $out/lib/libboost_regex.dylib \
       $out/lib/libnixexpr.dylib
     ''}
+  '' + lib.optionalString enableInternalAPIDocs ''
+    mkdir -p $out/nix-support
+    echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products
   '';
 
-  doInstallCheck = finalAttrs.doCheck;
+  doInstallCheck = attrs.doInstallCheck;
+
   installCheckFlags = "sysconfdir=$(out)/etc";
   installCheckTarget = "installcheck"; # work around buggy detection in stdenv
 
+  # Needed for tests if we are not doing a build, but testing existing
+  # built Nix.
+  preInstallCheck = lib.optionalString (! doBuild) ''
+    mkdir -p src/nix-channel
+  '';
+
   separateDebugInfo = !stdenv.hostPlatform.isStatic;
 
   strictDeps = true;
 
   hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
 
-  passthru ={
+  passthru = {
     inherit filesets;
 
     perl-bindings = callPackage ./perl {
@@ -258,6 +318,25 @@ in {
     };
   };
 
-  meta.platforms = lib.platforms.unix;
-  meta.mainProgram = "nix";
+  meta = {
+    platforms = lib.platforms.unix;
+    mainProgram = "nix";
+    broken = !(lib.all (a: a) [
+      (installUnitTests -> doBuild)
+      (doCheck -> doBuild)
+    ]);
+  };
+
+} // lib.optionalAttrs withCoverageChecks {
+  lcovFilter = [ "*/boost/*" "*-tab.*" ];
+
+  hardeningDisable = ["fortify"];
+
+  NIX_CFLAGS_COMPILE = "-DCOVERAGE=1";
+
+  dontInstall = false;
+} // lib.optionalAttrs (test-daemon != null) {
+  NIX_DAEMON_PACKAGE = test-daemon;
+} // lib.optionalAttrs (test-client != null) {
+  NIX_CLIENT_PACKAGE = test-client;
 })
diff --git a/test-nix-versions.nix b/test-nix-versions.nix
index 15f6cd8d0..bda4621a1 100644
--- a/test-nix-versions.nix
+++ b/test-nix-versions.nix
@@ -6,45 +6,10 @@
 }:
 
 stdenv.mkDerivation {
-  NIX_DAEMON_PACKAGE = daemon;
-  NIX_CLIENT_PACKAGE = client;
   name =
     "nix-tests"
     + lib.optionalString
       (lib.versionAtLeast daemon.version "2.4pre20211005" &&
        lib.versionAtLeast client.version "2.4pre20211005")
       "-${client.version}-against-${daemon.version}";
-
-  inherit (client)
-    version
-    VERSION_SUFFIX
-    nativeBuildInputs
-    buildInputs
-    propagatedBuildInputs
-    ;
-
-  src = fileset.toSource {
-    root = ./.;
-    fileset = with client.passthru.filesets;
-      fileset.intersect baseFiles (fileset.unions [
-        configureFiles
-        topLevelBuildFiles
-        functionalTestFiles
-      ]);
-  };
-
-  configureFlags = client.configureFlags # otherwise configure fails
-    ++ [ "--disable-build" ];
-
-  dontBuild = true;
-  doInstallCheck = true;
-
-  installPhase = ''
-    mkdir -p $out
-  '';
-
-  installCheckPhase = ''
-    mkdir -p src/nix-channel
-    make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES
-  '';
 }

From 3d47e024837a4340b1a0b6b6b8114e9e9e0c38a4 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 3 Dec 2023 16:48:50 -0500
Subject: [PATCH 141/421] WIP

---
 flake.nix   |  9 +++---
 package.nix | 90 +++++++++++++++++++++++++++++++----------------------
 2 files changed, 56 insertions(+), 43 deletions(-)

diff --git a/flake.nix b/flake.nix
index 44ce2d306..85ea1d052 100644
--- a/flake.nix
+++ b/flake.nix
@@ -153,7 +153,7 @@
                 then ""
                 else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";
 
-              sh = final.busybox-sandbox-shell or (final.busybox.override {
+              default-busybox-sandbox-shell = final.busybox.override {
                 useMusl = true;
                 enableStatic = true;
                 enableMinimal = true;
@@ -175,7 +175,7 @@
                   CONFIG_ASH_PRINTF y
                   CONFIG_ASH_TEST y
                 '';
-              });
+              };
 
               boehmgc = (final.boehmgc.override {
                 enableLargeConfig = true;
@@ -192,10 +192,10 @@
               inherit
                 boehmgc
                 fileset
-                sh
                 stdenv
                 versionSuffix
                 ;
+              busybox-sandbox-shell = final.busybox-sandbox-shell or default-busybox-sandbox-shell;
               boost = final.boost.override { enableIcu = false; };
               libgit2 = final.libgit2.overrideAttrs (attrs: {
                 src = libgit2;
@@ -277,9 +277,8 @@
 
         # API docs for Nix's unstable internal C++ interfaces.
         internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix {
+          inherit fileset;
           doBuild = false;
-          doCheck = false;
-          doInstallCheck = false;
           enableInternalAPIDocs = true;
         };
 
diff --git a/package.nix b/package.nix
index 39fee8472..15fe52b07 100644
--- a/package.nix
+++ b/package.nix
@@ -36,11 +36,12 @@
 , openssl
 , pkg-config
 , rapidcheck
-, sh
 , sqlite
 , util-linux
 , xz
 
+, busybox-sandbox-shell ? null
+
 # Configuration Options
 #
 # This probably seems like too many degrees of freedom, but it
@@ -50,11 +51,13 @@
 , pname ? "nix"
 
 , doBuild ? true
-, doCheck ? stdenv.buildPlatform.canExecute stdenv.hostPlatform
-, doInstallCheck ? stdenv.buildPlatform.canExecute stdenv.hostPlatform
+, doCheck ? __forDefaults.canRunInstalled
+, doInstallCheck ? __forDefaults.canRunInstalled
 
 , withCoverageChecks ? false
 
+# Whether to build the regular manual
+, enableManual ? __forDefaults.canRunInstalled
 # Whether to build the internal API docs, can be done separately from
 # everything else.
 , enableInternalAPIDocs ? false
@@ -62,16 +65,26 @@
 # Whether to install unit tests. This is useful when cross compiling
 # since we cannot run them natively during the build, but can do so
 # later.
-, installUnitTests ? stdenv.hostPlatform != stdenv.buildPlatform
+, installUnitTests ? __forDefaults.canRunInstalled
 
+# For running the functional tests against a pre-built Nix. Probably
+# want to use in conjunction with `doBuild = false;`.
 , test-daemon ? null
 , test-client ? null
-}:
+
+# Not a real argument, just the only way to approximate let-binding some
+# stuff for argument defaults.
+, __forDefaults ? {
+    canRunInstalled = doBuild && stdenv.buildPlatform.canExecute stdenv.hostPlatform;
+  }
+} @ attrs0:
 
 let
   version = lib.fileContents ./.version + versionSuffix;
-  canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
 
+  # selected attributes with defaults, will be used to define some
+  # things which should instead be gotten via `finalAttrs` in order to
+  # work with overriding.
   attrs = {
     inherit doBuild doCheck doInstallCheck;
   };
@@ -149,7 +162,11 @@ in {
 
   VERSION_SUFFIX = versionSuffix;
 
-  outputs = [ "out" "dev" "doc" ]
+  outputs = [ "out" ]
+    ++ lib.optional doBuild "dev"
+    # If we are doing just build or just docs, the one thing will use
+    # "out". We only need additional outputs if we are doing both.
+    ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs)) "doc"
     ++ lib.optional installUnitTests "check";
 
   nativeBuildInputs = [
@@ -164,10 +181,11 @@ in {
     pkg-config
   ]
   ++ lib.optional stdenv.hostPlatform.isLinux util-linux
-  # Official releases don't have rl-next, so we don't need to compile a changelog
+  # Official releases don't have rl-next, so we don't need to compile a
+  # changelog
   ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d;
 
-  buildInputs = [
+  buildInputs = lib.optionals doBuild [
     boost
     brotli
     bzip2
@@ -180,19 +198,14 @@ in {
     openssl
     sqlite
     xz
-
-    # These could be checkInputs but the configure phase fails w/o them
-    gtest
-    rapidcheck
-  ]
-  ++ lib.optional stdenv.isLinux libseccomp
-  ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid
-  # There have been issues building these dependencies
-  ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin))
-    (aws-sdk-cpp.override {
-      apis = ["s3" "transfer"];
-      customMemoryManagement = false;
-    })
+  ] ++ lib.optional stdenv.isLinux libseccomp
+    ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid
+    # There have been issues building these dependencies
+    ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin))
+      (aws-sdk-cpp.override {
+        apis = ["s3" "transfer"];
+        customMemoryManagement = false;
+      })
   ;
 
   propagatedBuildInputs = [
@@ -204,7 +217,8 @@ in {
   doCheck = attrs.doCheck;
 
   checkInputs = [
-    # see buildInputs. The configure script always wants its test libs
+    gtest
+    rapidcheck
   ];
 
   nativeCheckInputs = [
@@ -242,17 +256,17 @@ in {
     (lib.enableFeature doBuild "build")
     (lib.enableFeature anySortOfTesting "test")
     (lib.enableFeature enableInternalAPIDocs "internal-api-docs")
-    (lib.enableFeature canRunInstalled "doc-gen")
+    (lib.enableFeature enableManual "doc-gen")
     (lib.enableFeature installUnitTests "install-unit-tests")
   ] ++ lib.optionals installUnitTests [
     "--with-check-bin-dir=${builtins.placeholder "check"}/bin"
     "--with-check-lib-dir=${builtins.placeholder "check"}/lib"
-  ] ++ lib.optionals stdenv.isLinux [
+  ] ++ lib.optionals (doBuild && stdenv.isLinux) [
     "--with-boost=${boost}/lib"
-    "--with-sandbox-shell=${sh}/bin/busybox"
-  ] ++ lib.optional (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux"))
+    "--with-sandbox-shell=${busybox-sandbox-shell}/bin/busybox"
+  ] ++ lib.optional (doBuild && stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux"))
        "LDFLAGS=-fuse-ld=gold"
-    ++ lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell"
+    ++ lib.optional (doBuild && stdenv.hostPlatform.isStatic) "--enable-embedded-sandbox-shell"
     ++ lib.optional buildUnitTests "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include";
 
   enableParallelBuilding = true;
@@ -271,14 +285,14 @@ in {
     mkdir -p $out
   '';
 
-  postInstall = lib.optionalString doBuild ''
-    mkdir -p $doc/nix-support
-    echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
-    ${lib.optionalString stdenv.hostPlatform.isStatic ''
+  postInstall = lib.optionalString doBuild (
+    ''
+      mkdir -p $doc/nix-support
+      echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
+    '' + lib.optionalString stdenv.hostPlatform.isStatic ''
       mkdir -p $out/nix-support
       echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
-    ''}
-    ${lib.optionalString stdenv.isDarwin ''
+    '' + lib.optionalString stdenv.isDarwin ''
       install_name_tool \
       -change ${boost}/lib/libboost_context.dylib \
       $out/lib/libboost_context.dylib \
@@ -287,10 +301,10 @@ in {
       -change ${boost}/lib/libboost_regex.dylib \
       $out/lib/libboost_regex.dylib \
       $out/lib/libnixexpr.dylib
-    ''}
-  '' + lib.optionalString enableInternalAPIDocs ''
-    mkdir -p $out/nix-support
-    echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products
+    ''
+  ) + lib.optionalString enableInternalAPIDocs ''
+    mkdir -p ''${!outputDoc}/nix-support
+    echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products
   '';
 
   doInstallCheck = attrs.doInstallCheck;

From c71d987553530dcf02bcd7bf4c682634d7e5b6be Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 3 Dec 2023 17:12:38 -0500
Subject: [PATCH 142/421] Fix incorrect flag name

---
 package.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.nix b/package.nix
index d2498ade2..f688cc819 100644
--- a/package.nix
+++ b/package.nix
@@ -254,7 +254,7 @@ in {
   configureFlags = [
     "--sysconfdir=/etc"
     (lib.enableFeature doBuild "build")
-    (lib.enableFeature anySortOfTesting "test")
+    (lib.enableFeature anySortOfTesting "tests")
     (lib.enableFeature enableInternalAPIDocs "internal-api-docs")
     (lib.enableFeature enableManual "doc-gen")
     (lib.enableFeature installUnitTests "install-unit-tests")

From 7b51086d736f8cf983744510ff40e5afbc313079 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 3 Dec 2023 18:12:05 -0500
Subject: [PATCH 143/421] More fixes

---
 coverage.nix | 29 ----------------
 flake.nix    | 97 ++++++++++++++++++++++++++++++----------------------
 package.nix  | 61 +++++++++++++++++++++------------
 3 files changed, 95 insertions(+), 92 deletions(-)
 delete mode 100644 coverage.nix

diff --git a/coverage.nix b/coverage.nix
deleted file mode 100644
index 2c5e4a06d..000000000
--- a/coverage.nix
+++ /dev/null
@@ -1,29 +0,0 @@
-{ lib
-, releaseTools
-, nix
-, stdenv
-}:
-
-let
-  inherit (nix) version;
-
-in
-
-releaseTools.coverageAnalysis {
-  name = "nix-coverage-${version}";
-
-  inherit (nix)
-    src
-    buildInputs
-    nativeBuildInputs
-    propagatedBuildInputs
-    configureFlags
-    makeFlags
-    installFlags
-    doInstallCheck
-    installCheckFlags
-    installCheckTarget
-    ;
-
-  enableParallelBuilding = true;
-}
diff --git a/flake.nix b/flake.nix
index aafcfd71b..fab8c45be 100644
--- a/flake.nix
+++ b/flake.nix
@@ -123,8 +123,20 @@
           '';
 
       testNixVersions = pkgs: client: daemon:
-        pkgs.callPackage ./test-nix-versions.nix {
-          inherit client daemon fileset;
+        pkgs.callPackage ./package.nix {
+          pname =
+            "nix-tests"
+            + lib.optionalString
+              (lib.versionAtLeast daemon.version "2.4pre20211005" &&
+               lib.versionAtLeast client.version "2.4pre20211005")
+              "-${client.version}-against-${daemon.version}";
+
+          inherit fileset;
+
+          test-client = client;
+          test-daemon = daemon;
+
+          doBuild = false;
         };
 
       binaryTarball = nix: pkgs: pkgs.callPackage ./binary-tarball.nix {
@@ -134,10 +146,6 @@
       overlayFor = getStdenv: final: prev:
         let
           stdenv = getStdenv final;
-
-          lowdown-nix = final.callPackage ./lowdown.nix {
-            inherit lowdown-src stdenv;
-          };
         in
         {
           nixStable = prev.nix;
@@ -145,6 +153,41 @@
           # Forward from the previous stage as we don’t want it to pick the lowdown override
           inherit (prev) nixUnstable;
 
+          default-busybox-sandbox-shell = final.busybox.override {
+            useMusl = true;
+            enableStatic = true;
+            enableMinimal = true;
+            extraConfig = ''
+              CONFIG_FEATURE_FANCY_ECHO y
+              CONFIG_FEATURE_SH_MATH y
+              CONFIG_FEATURE_SH_MATH_64 y
+
+              CONFIG_ASH y
+              CONFIG_ASH_OPTIMIZE_FOR_SIZE y
+
+              CONFIG_ASH_ALIAS y
+              CONFIG_ASH_BASH_COMPAT y
+              CONFIG_ASH_CMDCMD y
+              CONFIG_ASH_ECHO y
+              CONFIG_ASH_GETOPTS y
+              CONFIG_ASH_INTERNAL_GLOB y
+              CONFIG_ASH_JOB_CONTROL y
+              CONFIG_ASH_PRINTF y
+              CONFIG_ASH_TEST y
+            '';
+          };
+
+          lowdown-nix = final.callPackage ./lowdown.nix {
+            inherit lowdown-src stdenv;
+          };
+
+          libgit2-nix = final.libgit2.overrideAttrs (attrs: {
+            src = libgit2;
+            version = libgit2.lastModifiedDate;
+            cmakeFlags = attrs.cmakeFlags or []
+              ++ [ "-DUSE_SSH=exec" ];
+          });
+
           nix =
             let
               officialRelease = false;
@@ -153,30 +196,6 @@
                 then ""
                 else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";
 
-              default-busybox-sandbox-shell = final.busybox.override {
-                useMusl = true;
-                enableStatic = true;
-                enableMinimal = true;
-                extraConfig = ''
-                  CONFIG_FEATURE_FANCY_ECHO y
-                  CONFIG_FEATURE_SH_MATH y
-                  CONFIG_FEATURE_SH_MATH_64 y
-
-                  CONFIG_ASH y
-                  CONFIG_ASH_OPTIMIZE_FOR_SIZE y
-
-                  CONFIG_ASH_ALIAS y
-                  CONFIG_ASH_BASH_COMPAT y
-                  CONFIG_ASH_CMDCMD y
-                  CONFIG_ASH_ECHO y
-                  CONFIG_ASH_GETOPTS y
-                  CONFIG_ASH_INTERNAL_GLOB y
-                  CONFIG_ASH_JOB_CONTROL y
-                  CONFIG_ASH_PRINTF y
-                  CONFIG_ASH_TEST y
-                '';
-              };
-
               boehmgc = (final.boehmgc.override {
                 enableLargeConfig = true;
               }).overrideAttrs(o: {
@@ -195,18 +214,11 @@
                 stdenv
                 versionSuffix
                 ;
-              busybox-sandbox-shell = final.busybox-sandbox-shell or default-busybox-sandbox-shell;
-              libgit2 = final.libgit2.overrideAttrs (attrs: {
-                src = libgit2;
-                version = libgit2.lastModifiedDate;
-                cmakeFlags = attrs.cmakeFlags or []
-                  ++ [ "-DUSE_SSH=exec" ];
-              });
-              lowdown = lowdown-nix;
               officialRelease = false;
+              libgit2 = final.libgit2-nix;
+              lowdown = final.lowdown-nix;
+              busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
             };
-
-            inherit lowdown-nix;
           };
 
     in {
@@ -272,7 +284,10 @@
         dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage);
 
         # Line coverage analysis.
-        coverage = nixpkgsFor.x86_64-linux.native.callPackage ./coverage.nix {};
+        coverage = nixpkgsFor.x86_64-linux.native.nix.override {
+          pname = "nix-coverage";
+          withCoverageChecks = true;
+        };
 
         # API docs for Nix's unstable internal C++ interfaces.
         internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix {
diff --git a/package.nix b/package.nix
index f688cc819..0758f989e 100644
--- a/package.nix
+++ b/package.nix
@@ -2,9 +2,6 @@
 , callPackage
 , stdenv
 , releaseTools
-, versionSuffix ? ""
-, officialRelease ? false
-, buildUnreleasedNotes ? false
 , autoconf-archive
 , autoreconfHook
 , aws-sdk-cpp
@@ -43,21 +40,25 @@
 , busybox-sandbox-shell ? null
 
 # Configuration Options
-#
+#:
 # This probably seems like too many degrees of freedom, but it
 # faithfully reflects how the underlying configure + make build system
 # work. The top-level flake.nix will choose useful combinations.
 
 , pname ? "nix"
 
+, versionSuffix ? ""
+, officialRelease ? false
+
 , doBuild ? true
 , doCheck ? __forDefaults.canRunInstalled
-, doInstallCheck ? __forDefaults.canRunInstalled
+, doInstallCheck ? test-client != null || __forDefaults.canRunInstalled
 
 , withCoverageChecks ? false
 
 # Whether to build the regular manual
 , enableManual ? __forDefaults.canRunInstalled
+, buildUnreleasedNotes ? false
 # Whether to build the internal API docs, can be done separately from
 # everything else.
 , enableInternalAPIDocs ? false
@@ -115,7 +116,11 @@ let
 
   mkDerivation =
     if withCoverageChecks
-    then releaseTools.coverageAnalysis
+    then
+      # TODO support `finalAttrs` args function in
+      # `releaseTools.coverageAnalysis`.
+      argsFun:
+         releaseTools.coverageAnalysis (let args = argsFun args; in args)
     else stdenv.mkDerivation;
 in
 
@@ -146,6 +151,7 @@ in {
         fileset = fileset.intersect filesets.baseFiles (fileset.unions ([
           filesets.configureFiles
           filesets.topLevelBuildFiles
+          ./doc/internal-api
         ] ++ lib.optionals doBuild [
           ./boehmgc-coroutine-sp-fallback.diff
           ./doc
@@ -170,20 +176,24 @@ in {
     ++ lib.optional installUnitTests "check";
 
   nativeBuildInputs = [
-    bison
-    flex
-    (lib.getBin lowdown)
-    jq # Also for custom mdBook preprocessor.
-    mdbook
-    mdbook-linkcheck
     autoconf-archive
     autoreconfHook
     pkg-config
-  ]
-  ++ lib.optional stdenv.hostPlatform.isLinux util-linux
-  # Official releases don't have rl-next, so we don't need to compile a
-  # changelog
-  ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d;
+  ] ++ lib.optionals doBuild [
+    bison
+    flex
+  ] ++ lib.optionals enableManual [
+    (lib.getBin lowdown)
+    mdbook
+    mdbook-linkcheck
+  ] ++ lib.optionals (doInstallCheck || enableManual) [
+    jq # Also for custom mdBook preprocessor.
+  ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux
+    # Official releases don't have rl-next, so we don't need to compile a
+    # changelog
+    ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d
+    ++ lib.optional enableInternalAPIDocs doxygen
+  ;
 
   buildInputs = lib.optionals doBuild [
     boost
@@ -225,13 +235,11 @@ in {
     git
     mercurial
     openssh
-  ] ++ lib.optionals enableInternalAPIDocs [
-    doxygen
   ];
 
   disallowedReferences = [ boost ];
 
-  preConfigure = lib.optionalString (! stdenv.hostPlatform.isStatic) ''
+  preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) ''
     # Copy libboost_context so we don't get all of Boost in our closure.
     # https://github.com/NixOS/nixpkgs/issues/45462
     mkdir -p $out/lib
@@ -307,7 +315,14 @@ in {
   doInstallCheck = attrs.doInstallCheck;
 
   installCheckFlags = "sysconfdir=$(out)/etc";
-  installCheckTarget = "installcheck"; # work around buggy detection in stdenv
+  # work around buggy detection in stdenv
+  installCheckTarget = "installcheck";
+
+  # work around weird bug where it doesn't want to do anything
+  installCheckPhase = if (!doBuild && doInstallCheck) then ''
+    mkdir -p src/nix-channel
+    make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES
+  '' else null;
 
   # Needed for tests if we are not doing a build, but testing existing
   # built Nix.
@@ -317,7 +332,9 @@ in {
 
   separateDebugInfo = !stdenv.hostPlatform.isStatic;
 
-  strictDeps = true;
+  # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated
+  # to work with `strictDeps`.
+  strictDeps = !withCoverageChecks;
 
   hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
 

From c160c6251566e758dd4d8fd409df3fa3b2f832b9 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 3 Dec 2023 18:12:22 -0500
Subject: [PATCH 144/421] Fix underlying build system so `--disable-build`
 works better

- Internal API docs once again work

- configure skips checks for a bunch of things it doesn't need
---
 Makefile     |  2 +-
 configure.ac | 50 ++++++++++++++++++++++++++++----------------------
 2 files changed, 29 insertions(+), 23 deletions(-)

diff --git a/Makefile b/Makefile
index eea297c89..0b2b408ca 100644
--- a/Makefile
+++ b/Makefile
@@ -61,7 +61,7 @@ include mk/lib.mk
 # by the library. Rules are not "lazy" like variables, unfortunately.
 ifeq ($(ENABLE_BUILD), yes)
 $(eval $(call include-sub-makefile, doc/manual/local.mk))
-$(eval $(call include-sub-makefile, doc/internal-api/local.mk))
 endif
+$(eval $(call include-sub-makefile, doc/internal-api/local.mk))
 
 GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++2a -I src
diff --git a/configure.ac b/configure.ac
index f8b937eb5..f9ad3c840 100644
--- a/configure.ac
+++ b/configure.ac
@@ -122,7 +122,6 @@ AC_PATH_PROG(flex, flex, false)
 AC_PATH_PROG(bison, bison, false)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(lsof, lsof, lsof)
-NEED_PROG(jq, jq)
 
 
 AC_SUBST(coreutils, [$(dirname $(type -p cat))])
@@ -133,6 +132,30 @@ AC_ARG_WITH(store-dir, AS_HELP_STRING([--with-store-dir=PATH],[path of the Nix s
 AC_SUBST(storedir)
 
 
+# Running the functional tests without building Nix is useful for testing
+# different pre-built versions of Nix against each other.
+AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]),
+  ENABLE_BUILD=$enableval, ENABLE_BUILD=yes)
+AC_SUBST(ENABLE_BUILD)
+
+# Building without tests is useful for bootstrapping with a smaller footprint
+# or running the tests in a separate derivation. Otherwise, we do compile and
+# run them.
+AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]),
+  ENABLE_TESTS=$enableval, ENABLE_TESTS=yes)
+AC_SUBST(ENABLE_TESTS)
+
+# Building without API docs is the default as Nix' C++ interfaces are internal and unstable.
+AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]),
+  internal_api_docs=$enableval, internal_api_docs=no)
+AC_SUBST(internal_api_docs)
+
+AS_IF(
+  [test "$ENABLE_BUILD" == "yes" || test "$ENABLE_TEST" == "yes"],
+  [NEED_PROG(jq, jq)])
+
+AS_IF([test "$ENABLE_BUILD" == "yes"],[
+
 # Look for boost, a required dependency.
 # Note that AX_BOOST_BASE only exports *CPP* BOOST_CPPFLAGS, no CXX flags,
 # and CPPFLAGS are not passed to the C++ compiler automatically.
@@ -155,18 +178,6 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then
     LDFLAGS="-latomic $LDFLAGS"
 fi
 
-# Running the functional tests without building Nix is useful for testing
-# different pre-built versions of Nix against each other.
-AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]),
-  ENABLE_BUILD=$enableval, ENABLE_BUILD=yes)
-AC_SUBST(ENABLE_BUILD)
-# Building without tests is useful for bootstrapping with a smaller footprint
-# or running the tests in a separate derivation. Otherwise, we do compile and
-# run them.
-AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]),
-  ENABLE_TESTS=$enableval, ENABLE_TESTS=yes)
-AC_SUBST(ENABLE_TESTS)
-
 AC_ARG_ENABLE(install-unit-tests, AS_HELP_STRING([--enable-install-unit-tests],[Install the unit tests for running later (default no)]),
   INSTALL_UNIT_TESTS=$enableval, INSTALL_UNIT_TESTS=no)
 AC_SUBST(INSTALL_UNIT_TESTS)
@@ -179,11 +190,6 @@ AC_ARG_WITH(check-lib-dir, AS_HELP_STRING([--with-check-lib-dir=PATH],[path to i
   checklibdir=$withval, checklibdir=$libdir)
 AC_SUBST(checklibdir)
 
-# Building without API docs is the default as Nix' C++ interfaces are internal and unstable.
-AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]),
-  internal_api_docs=$enableval, internal_api_docs=no)
-AC_SUBST(internal_api_docs)
-
 # LTO is currently broken with clang for unknown reasons; ld segfaults in the llvm plugin
 AC_ARG_ENABLE(lto, AS_HELP_STRING([--enable-lto],[Enable LTO (only supported with GCC) [default=no]]),
   lto=$enableval, lto=no)
@@ -310,8 +316,7 @@ if test "$gc" = yes; then
   AC_DEFINE(HAVE_BOEHMGC, 1, [Whether to use the Boehm garbage collector.])
 fi
 
-
-if test "$ENABLE_TESTS" = yes; then
+AS_IF([test "$ENABLE_TESTS" == "yes"],[
 
 # Look for gtest.
 PKG_CHECK_MODULES([GTEST], [gtest_main])
@@ -338,12 +343,11 @@ AC_LINK_IFELSE([
   [AC_MSG_ERROR([librapidcheck is not found.])])
 AC_LANG_POP(C++)
 
-fi
+])
 
 # Look for nlohmann/json.
 PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9])
 
-
 # documentation generation switch
 AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation generation]),
   doc_generate=$enableval, doc_generate=yes)
@@ -388,6 +392,8 @@ if test "$embedded_sandbox_shell" = yes; then
   AC_DEFINE(HAVE_EMBEDDED_SANDBOX_SHELL, 1, [Include the sandbox shell in the Nix binary.])
 fi
 
+])
+
 
 # Expand all variables in config.status.
 test "$prefix" = NONE && prefix=$ac_default_prefix

From 7a7ad7c84b4dd37331a8f8889b02c94540522dbc Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 3 Dec 2023 18:14:36 -0500
Subject: [PATCH 145/421] Remove uneeded file

---
 test-nix-versions.nix | 15 ---------------
 1 file changed, 15 deletions(-)
 delete mode 100644 test-nix-versions.nix

diff --git a/test-nix-versions.nix b/test-nix-versions.nix
deleted file mode 100644
index bda4621a1..000000000
--- a/test-nix-versions.nix
+++ /dev/null
@@ -1,15 +0,0 @@
-{ lib
-, fileset
-, stdenv
-, client
-, daemon
-}:
-
-stdenv.mkDerivation {
-  name =
-    "nix-tests"
-    + lib.optionalString
-      (lib.versionAtLeast daemon.version "2.4pre20211005" &&
-       lib.versionAtLeast client.version "2.4pre20211005")
-      "-${client.version}-against-${daemon.version}";
-}

From e275f0adfb6b3f360f10f5adcf140c17edc58cc6 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 3 Dec 2023 18:16:07 -0500
Subject: [PATCH 146/421] Move `binary-tarball.nix` to scripts dir

---
 flake.nix                                        |  2 +-
 binary-tarball.nix => scripts/binary-tarball.nix | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)
 rename binary-tarball.nix => scripts/binary-tarball.nix (85%)

diff --git a/flake.nix b/flake.nix
index fab8c45be..5c6ad3bc7 100644
--- a/flake.nix
+++ b/flake.nix
@@ -139,7 +139,7 @@
           doBuild = false;
         };
 
-      binaryTarball = nix: pkgs: pkgs.callPackage ./binary-tarball.nix {
+      binaryTarball = nix: pkgs: pkgs.callPackage ./scripts/binary-tarball.nix {
         inherit nix;
       };
 
diff --git a/binary-tarball.nix b/scripts/binary-tarball.nix
similarity index 85%
rename from binary-tarball.nix
rename to scripts/binary-tarball.nix
index 0053abbca..32e811c94 100644
--- a/binary-tarball.nix
+++ b/scripts/binary-tarball.nix
@@ -21,18 +21,18 @@ in
 
 runCommand "nix-binary-tarball-${version}" env ''
   cp ${installerClosureInfo}/registration $TMPDIR/reginfo
-  cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
-  substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
+  cp ${./create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
+  substitute ${./install-nix-from-closure.sh} $TMPDIR/install \
     --subst-var-by nix ${nix} \
     --subst-var-by cacert ${cacert}
 
-  substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
+  substitute ${./install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
     --subst-var-by nix ${nix} \
     --subst-var-by cacert ${cacert}
-  substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
+  substitute ${./install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
     --subst-var-by nix ${nix} \
     --subst-var-by cacert ${cacert}
-  substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
+  substitute ${./install-multi-user.sh} $TMPDIR/install-multi-user \
     --subst-var-by nix ${nix} \
     --subst-var-by cacert ${cacert}
 

From 60fe4ddaa1801b37a044a2c96071d96739bd26c0 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 3 Dec 2023 18:17:47 -0500
Subject: [PATCH 147/421] Expose `boehmgc-nix` in overlay

---
 flake.nix | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/flake.nix b/flake.nix
index 5c6ad3bc7..78fc88bed 100644
--- a/flake.nix
+++ b/flake.nix
@@ -188,6 +188,17 @@
               ++ [ "-DUSE_SSH=exec" ];
           });
 
+          boehmgc-nix = (final.boehmgc.override {
+            enableLargeConfig = true;
+          }).overrideAttrs(o: {
+            patches = (o.patches or []) ++ [
+              ./boehmgc-coroutine-sp-fallback.diff
+
+              # https://github.com/ivmai/bdwgc/pull/586
+              ./boehmgc-traceable_allocator-public.diff
+            ];
+          });
+
           nix =
             let
               officialRelease = false;
@@ -196,25 +207,14 @@
                 then ""
                 else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";
 
-              boehmgc = (final.boehmgc.override {
-                enableLargeConfig = true;
-              }).overrideAttrs(o: {
-                patches = (o.patches or []) ++ [
-                  ./boehmgc-coroutine-sp-fallback.diff
-
-                  # https://github.com/ivmai/bdwgc/pull/586
-                  ./boehmgc-traceable_allocator-public.diff
-                ];
-              });
-
             in final.callPackage ./package.nix {
               inherit
-                boehmgc
                 fileset
                 stdenv
                 versionSuffix
                 ;
               officialRelease = false;
+              boehmgc = final.boehmgc-nix;
               libgit2 = final.libgit2-nix;
               lowdown = final.lowdown-nix;
               busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;

From 77003a4f0c380929f18b71476b9e7f9cd4009458 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 3 Dec 2023 18:29:15 -0500
Subject: [PATCH 148/421] Factor out the installer script

---
 flake.nix             | 43 +++++++++++--------------------------------
 scripts/installer.nix | 35 +++++++++++++++++++++++++++++++++++
 2 files changed, 46 insertions(+), 32 deletions(-)
 create mode 100644 scripts/installer.nix

diff --git a/flake.nix b/flake.nix
index 78fc88bed..ecd0381a2 100644
--- a/flake.nix
+++ b/flake.nix
@@ -89,38 +89,17 @@
         });
 
       installScriptFor = systems:
-        with nixpkgsFor.x86_64-linux.native;
-        runCommand "installer-script"
-          { buildInputs = [ nix ];
-          }
-          ''
-            mkdir -p $out/nix-support
-
-            # Converts /nix/store/50p3qk8k...-nix-2.4pre20201102_550e11f/bin/nix to 50p3qk8k.../bin/nix.
-            tarballPath() {
-              # Remove the store prefix
-              local path=''${1#${builtins.storeDir}/}
-              # Get the path relative to the derivation root
-              local rest=''${path#*/}
-              # Get the derivation hash
-              local drvHash=''${path%%-*}
-              echo "$drvHash/$rest"
-            }
-
-            substitute ${./scripts/install.in} $out/install \
-              ${pkgs.lib.concatMapStrings
-                (system: let
-                    tarball = if builtins.elem system crossSystems then self.hydraJobs.binaryTarballCross.x86_64-linux.${system} else self.hydraJobs.binaryTarball.${system};
-                  in '' \
-                  --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \
-                  --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \
-                  ''
-                )
-                systems
-              } --replace '@nixVersion@' ${version}
-
-            echo "file installer $out/install" >> $out/nix-support/hydra-build-products
-          '';
+        nixpkgsFor.x86_64-linux.native.callPackage ./scripts/installer.nix {
+          systemTarballPairs = map
+            (system: {
+              inherit system;
+              tarball =
+                if builtins.elem system crossSystems
+                then self.hydraJobs.binaryTarballCross.x86_64-linux.${system}
+                else self.hydraJobs.binaryTarball.${system};
+            })
+            systems;
+        };
 
       testNixVersions = pkgs: client: daemon:
         pkgs.callPackage ./package.nix {
diff --git a/scripts/installer.nix b/scripts/installer.nix
new file mode 100644
index 000000000..35d2d7fe6
--- /dev/null
+++ b/scripts/installer.nix
@@ -0,0 +1,35 @@
+{ lib
+, runCommand
+, nix
+, systemTarballPairs
+}:
+
+runCommand "installer-script" {
+  buildInputs = [ nix ];
+} ''
+  mkdir -p $out/nix-support
+
+  # Converts /nix/store/50p3qk8k...-nix-2.4pre20201102_550e11f/bin/nix to 50p3qk8k.../bin/nix.
+  tarballPath() {
+    # Remove the store prefix
+    local path=''${1#${builtins.storeDir}/}
+    # Get the path relative to the derivation root
+    local rest=''${path#*/}
+    # Get the derivation hash
+    local drvHash=''${path%%-*}
+    echo "$drvHash/$rest"
+  }
+
+  substitute ${./install.in} $out/install \
+    ${lib.concatMapStrings
+      ({ system, tarball }:
+        '' \
+        --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \
+        --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \
+        ''
+      )
+      systemTarballPairs
+    } --replace '@nixVersion@' ${nix.version}
+
+  echo "file installer $out/install" >> $out/nix-support/hydra-build-products
+''

From f58615518c1284d4dbe4655246d4d5d6e9b2befe Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 3 Dec 2023 18:39:33 -0500
Subject: [PATCH 149/421] Add documenting comments to `package.nix`

---
 package.nix | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/package.nix b/package.nix
index 0758f989e..7f0d78b5c 100644
--- a/package.nix
+++ b/package.nix
@@ -43,22 +43,37 @@
 #:
 # This probably seems like too many degrees of freedom, but it
 # faithfully reflects how the underlying configure + make build system
-# work. The top-level flake.nix will choose useful combinations.
+# work. The top-level flake.nix will choose useful combinations of these
+# options to CI.
 
 , pname ? "nix"
 
 , versionSuffix ? ""
 , officialRelease ? false
 
+# Whether to build Nix. Useful to skip for tasks like (a) just
+# generating API docs or (b) testing existing pre-built versions of Nix
 , doBuild ? true
+
+# Run the unit tests as part of the build. See `installUnitTests` for an
+# alternative to this.
 , doCheck ? __forDefaults.canRunInstalled
+
+# Run the functional tests as part of the build.
 , doInstallCheck ? test-client != null || __forDefaults.canRunInstalled
 
+# Check test coverage of Nix. Probably want to use with with at least
+# one of `doCHeck` or `doInstallCheck` enabled.
 , withCoverageChecks ? false
 
 # Whether to build the regular manual
 , enableManual ? __forDefaults.canRunInstalled
+
+# Whether to compile `rl-next.md`, the release notes for the next
+# not-yet-released version of Nix in the manul, from the individual
+# change log entries in the directory.
 , buildUnreleasedNotes ? false
+
 # Whether to build the internal API docs, can be done separately from
 # everything else.
 , enableInternalAPIDocs ? false
@@ -350,8 +365,13 @@ in {
     platforms = lib.platforms.unix;
     mainProgram = "nix";
     broken = !(lib.all (a: a) [
+      # We cannot run or install unit tests if we don't build them or
+      # Nix proper (which they depend on).
       (installUnitTests -> doBuild)
       (doCheck -> doBuild)
+      # We have to build the manual to build unreleased notes, as those
+      # are part of the manual
+      (buildUnreleasedNotes -> enableManual)
     ]);
   };
 

From a5a45e64e18de3eb827ca83c7356dc8a088be125 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 3 Dec 2023 18:45:15 -0500
Subject: [PATCH 150/421] Don't expose file sets anymore

---
 package.nix | 2 --
 1 file changed, 2 deletions(-)

diff --git a/package.nix b/package.nix
index 7f0d78b5c..52050496c 100644
--- a/package.nix
+++ b/package.nix
@@ -354,8 +354,6 @@ in {
   hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
 
   passthru = {
-    inherit filesets;
-
     perl-bindings = callPackage ./perl {
       inherit fileset stdenv;
     };

From 7e2b1cce6abec48f85c8bc056da0ca991dfe7b32 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 3 Dec 2023 18:47:54 -0500
Subject: [PATCH 151/421] Slap on `perl-bindings` in the caller

The Perl bindings are not part of Nix, but a downstream package, so they
don't belong in `package.nix`.

They don't really belong as an attribute on `nix` either, but we can
just leave that interface as is for now.
---
 flake.nix   | 9 +++++++++
 package.nix | 7 -------
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/flake.nix b/flake.nix
index ecd0381a2..c92f717d5 100644
--- a/flake.nix
+++ b/flake.nix
@@ -197,7 +197,16 @@
               libgit2 = final.libgit2-nix;
               lowdown = final.lowdown-nix;
               busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
+            } // {
+              # this is a proper separate downstream package, but put
+              # here also for back compat reasons.
+              perl-bindings = final.nix-perl-bindings;
             };
+
+            nix-perl-bindings = final.callPackage ./perl {
+              inherit fileset stdenv;
+            };
+
           };
 
     in {
diff --git a/package.nix b/package.nix
index 52050496c..f6219e58a 100644
--- a/package.nix
+++ b/package.nix
@@ -1,5 +1,4 @@
 { lib
-, callPackage
 , stdenv
 , releaseTools
 , autoconf-archive
@@ -353,12 +352,6 @@ in {
 
   hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
 
-  passthru = {
-    perl-bindings = callPackage ./perl {
-      inherit fileset stdenv;
-    };
-  };
-
   meta = {
     platforms = lib.platforms.unix;
     mainProgram = "nix";

From 6e0656c66c1052bcbab204140c6b3dec81f3ab15 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 3 Dec 2023 18:53:05 -0500
Subject: [PATCH 152/421] Add another configure flag assertion

---
 package.nix | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/package.nix b/package.nix
index f6219e58a..42f98a48c 100644
--- a/package.nix
+++ b/package.nix
@@ -363,6 +363,9 @@ in {
       # We have to build the manual to build unreleased notes, as those
       # are part of the manual
       (buildUnreleasedNotes -> enableManual)
+      # The build process for the manual currently requires extracting
+      # data from the Nix executable we are trying to document.
+      (enableManual -> doBuild)
     ]);
   };
 

From 14c26d642ebcff3fe45c8eb6719a213c63143fb3 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 3 Dec 2023 18:57:16 -0500
Subject: [PATCH 153/421] Clean up two comments

---
 package.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/package.nix b/package.nix
index 42f98a48c..96b9111f8 100644
--- a/package.nix
+++ b/package.nix
@@ -329,10 +329,10 @@ in {
   doInstallCheck = attrs.doInstallCheck;
 
   installCheckFlags = "sysconfdir=$(out)/etc";
-  # work around buggy detection in stdenv
+  # Work around buggy detection in stdenv.
   installCheckTarget = "installcheck";
 
-  # work around weird bug where it doesn't want to do anything
+  # Work around weird bug where it doesn't think there is a Makefile.
   installCheckPhase = if (!doBuild && doInstallCheck) then ''
     mkdir -p src/nix-channel
     make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES

From 2e5abc0fd0d5d45e125e1d981958149624268090 Mon Sep 17 00:00:00 2001
From: wh0 <wh0@users.noreply.github.com>
Date: Sun, 3 Dec 2023 17:18:58 -0800
Subject: [PATCH 154/421] tests: avoid a chroot store without sandbox support

---
 tests/functional/build-remote-trustless-should-fail-0.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/functional/build-remote-trustless-should-fail-0.sh b/tests/functional/build-remote-trustless-should-fail-0.sh
index fad1def59..3d4a4b097 100644
--- a/tests/functional/build-remote-trustless-should-fail-0.sh
+++ b/tests/functional/build-remote-trustless-should-fail-0.sh
@@ -4,6 +4,7 @@ enableFeatures "daemon-trust-override"
 
 restartDaemon
 
+requireSandboxSupport
 [[ $busybox =~ busybox ]] || skipTest "no busybox"
 
 unset NIX_STORE_DIR

From 3c310bde2e492c2dd8bdccdfd80076231905a429 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Tue, 14 Nov 2023 11:40:56 +0100
Subject: [PATCH 155/421] reword description for the `fetch-tree` experimental
 feature

without knowing a lot of context, it's not clear who "we" are in that
text. I'm also strongly opposed to adding procedural notes into
a reference manual; it just won't age well.

this change leaves a factual description of the experimental feature and
its purpose.
---
 src/libutil/experimental-features.cc | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index e4bdb8cb3..9b46fc5b0 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -80,12 +80,11 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
         .description = R"(
             Enable the use of the [`fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) built-in function in the Nix language.
 
-            `fetchTree` exposes a large suite of fetching functionality in a more systematic way.
+            `fetchTree` exposes a generic interface for fetching remote file system trees from different types of remote sources.
             The [`flakes`](#xp-feature-flakes) feature flag always enables `fetch-tree`.
+            This built-in was previously guarded by the `flakes` experimental feature because of that overlap.
 
-            This built-in was previously guarded by the `flakes` experimental feature because of that overlap,
-            but since the plan is to work on stabilizing this first (due 2024 Q1), we are putting it underneath a separate feature.
-            Once we've made the changes we want to make, enabling just this feature will serve as a "release candidate" --- allowing users to try out the functionality we want to stabilize and not any other functionality we don't yet want to, in isolation.
+            Enabling just this feature serves as a "release candidate", allowing users to try it out in isolation.
         )",
     },
     {

From 5fe2accb754249df6cb8f840330abfcf3bd26695 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Tue, 14 Nov 2023 11:44:34 +0100
Subject: [PATCH 156/421] fix up release note

---
 doc/manual/src/release-notes/rl-2.19.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md
index 4eecaf929..ba6eb9c64 100644
--- a/doc/manual/src/release-notes/rl-2.19.md
+++ b/doc/manual/src/release-notes/rl-2.19.md
@@ -18,7 +18,7 @@
 - `nix-shell` shebang lines now support single-quoted arguments.
 
 - `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree).
-  As described in the documentation for that feature, this is because we anticipate polishing it and then stabilizing it before the rest of flakes.
+  This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/contributing/experimental-features.md#xp-fetch-tree).
 
 - The interface for creating and updating lock files has been overhauled:
 

From 823512c1e705d1fce8dfb8cde65228364c9a8045 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 4 Dec 2023 22:09:10 +0000
Subject: [PATCH 157/421] Bump zeebe-io/backport-action from 2.1.1 to 2.2.0

Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 2.1.1 to 2.2.0.
- [Release notes](https://github.com/zeebe-io/backport-action/releases)
- [Commits](https://github.com/zeebe-io/backport-action/compare/v2.1.1...v2.2.0)

---
updated-dependencies:
- dependency-name: zeebe-io/backport-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/backport.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
index 975c90b91..85ddcfad3 100644
--- a/.github/workflows/backport.yml
+++ b/.github/workflows/backport.yml
@@ -21,7 +21,7 @@ jobs:
           fetch-depth: 0
       - name: Create backport PRs
         # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v2.1.1
+        uses: zeebe-io/backport-action@v2.2.0
         with:
           # Config README: https://github.com/zeebe-io/backport-action#backport-action
           github_token: ${{ secrets.GITHUB_TOKEN }}

From c446e5294dbc12729e7bc55ee10b40dbaeeaacf0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 4 Dec 2023 22:09:14 +0000
Subject: [PATCH 158/421] Bump cachix/install-nix-action from 23 to 24

Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 23 to 24.
- [Release notes](https://github.com/cachix/install-nix-action/releases)
- [Commits](https://github.com/cachix/install-nix-action/compare/v23...v24)

---
updated-dependencies:
- dependency-name: cachix/install-nix-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/ci.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index afe4dc2e3..34a23b5f2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -20,7 +20,7 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v23
+    - uses: cachix/install-nix-action@v24
       with:
         # The sandbox would otherwise be disabled by default on Darwin
         extra_nix_config: "sandbox = true"
@@ -62,7 +62,7 @@ jobs:
       with:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v23
+    - uses: cachix/install-nix-action@v24
       with:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
     - uses: cachix/cachix-action@v12
@@ -84,7 +84,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v23
+    - uses: cachix/install-nix-action@v24
       with:
         install_url: '${{needs.installer.outputs.installerURL}}'
         install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
@@ -114,7 +114,7 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v23
+    - uses: cachix/install-nix-action@v24
       with:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV

From e6a3cbfceb66e06184b625a3913a786f68e71a1f Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 4 Dec 2023 22:09:18 +0000
Subject: [PATCH 159/421] Bump cachix/cachix-action from 12 to 13

Bumps [cachix/cachix-action](https://github.com/cachix/cachix-action) from 12 to 13.
- [Release notes](https://github.com/cachix/cachix-action/releases)
- [Commits](https://github.com/cachix/cachix-action/compare/v12...v13)

---
updated-dependencies:
- dependency-name: cachix/cachix-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index afe4dc2e3..033832c9a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -25,7 +25,7 @@ jobs:
         # The sandbox would otherwise be disabled by default on Darwin
         extra_nix_config: "sandbox = true"
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/cachix-action@v12
+    - uses: cachix/cachix-action@v13
       if: needs.check_secrets.outputs.cachix == 'true'
       with:
         name: '${{ env.CACHIX_NAME }}'
@@ -65,7 +65,7 @@ jobs:
     - uses: cachix/install-nix-action@v23
       with:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
-    - uses: cachix/cachix-action@v12
+    - uses: cachix/cachix-action@v13
       with:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
@@ -119,7 +119,7 @@ jobs:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV
-    - uses: cachix/cachix-action@v12
+    - uses: cachix/cachix-action@v13
       if: needs.check_secrets.outputs.cachix == 'true'
       with:
         name: '${{ env.CACHIX_NAME }}'

From e488a43f457f3ef9dba92184428bbe5381fe2634 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 4 Dec 2023 22:09:25 +0000
Subject: [PATCH 160/421] Bump actions/labeler from 4 to 5

Bumps [actions/labeler](https://github.com/actions/labeler) from 4 to 5.
- [Release notes](https://github.com/actions/labeler/releases)
- [Commits](https://github.com/actions/labeler/compare/v4...v5)

---
updated-dependencies:
- dependency-name: actions/labeler
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/labels.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml
index d83cb4f18..34aa4e6bd 100644
--- a/.github/workflows/labels.yml
+++ b/.github/workflows/labels.yml
@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     if: github.repository_owner == 'NixOS'
     steps:
-    - uses: actions/labeler@v4
+    - uses: actions/labeler@v5
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         sync-labels: false

From 345f79d01676680f2d4ef8803790896a190c855b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 5 Dec 2023 15:14:28 +0100
Subject: [PATCH 161/421] Check that we can't follow symlinks outside of the
 allowed paths

---
 tests/functional/restricted.sh | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh
index b8deceacc..cb83c34b1 100644
--- a/tests/functional/restricted.sh
+++ b/tests/functional/restricted.sh
@@ -39,6 +39,15 @@ nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT -
 
 [[ $(nix eval --raw --impure --restrict-eval -I . --expr 'builtins.readFile "${import ./simple.nix}/hello"') == 'Hello World!' ]]
 
+# Check that we can't follow a symlink outside of the allowed paths.
+mkdir -p $TEST_ROOT/tunnel.d
+ln -sfn .. $TEST_ROOT/tunnel.d/tunnel
+echo foo > $TEST_ROOT/bar
+
+expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readFile <foo/tunnel/bar>" -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode"
+
+expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir <foo/tunnel>" -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode"
+
 # Check whether we can leak symlink information through directory traversal.
 traverseDir="$(pwd)/restricted-traverse-me"
 ln -sfn "$(pwd)/restricted-secret" "$(pwd)/restricted-innocent"

From 733333e87db391e4f832de65f0f49f60e50c45a4 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 1 Dec 2023 17:38:34 -0500
Subject: [PATCH 162/421] Including `config.h` also needs `$(buildprefix)`

Per the instruction in the manual, we want to run configure in a
different directory so that we can configure + build for multiple
platforms. That means `config.h` will be in the build directory. This is
just like `Makefile.config`, which already is used with
`$(buildprefix)`.
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index eea297c89..41f14ac92 100644
--- a/Makefile
+++ b/Makefile
@@ -64,4 +64,4 @@ $(eval $(call include-sub-makefile, doc/manual/local.mk))
 $(eval $(call include-sub-makefile, doc/internal-api/local.mk))
 endif
 
-GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++2a -I src
+GLOBAL_CXXFLAGS += -g -Wall -include $(buildprefix)config.h -std=c++2a -I src

From 83c067c0fa0cc5a2dca440e5c986afe40b163802 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 5 Dec 2023 23:02:59 +0100
Subject: [PATCH 163/421] PosixSourceAccessor: Don't follow any symlinks

All path components must not be symlinks now (so the user needs to
call `resolveSymlinks()` when needed).
---
 src/libexpr/parser.y                 | 11 +++++-----
 src/libexpr/primops.cc               | 30 ++++++++++++++--------------
 src/libutil/posix-source-accessor.cc | 27 +++++++++++++++++++++----
 src/libutil/posix-source-accessor.hh |  5 +++++
 src/nix-env/nix-env.cc               |  6 +++---
 src/nix-env/user-env.cc              |  2 +-
 tests/functional/restricted.sh       |  7 +++++--
 7 files changed, 58 insertions(+), 30 deletions(-)

diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 58fc580fc..16ad8af2e 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -692,16 +692,17 @@ SourcePath resolveExprPath(SourcePath path)
 
     /* If `path' is a symlink, follow it.  This is so that relative
        path references work. */
-    while (true) {
+    while (!path.path.isRoot()) {
         // Basic cycle/depth limit to avoid infinite loops.
         if (++followCount >= maxFollow)
             throw Error("too many symbolic links encountered while traversing the path '%s'", path);
-        if (path.lstat().type != InputAccessor::tSymlink) break;
-        path = {path.accessor, CanonPath(path.readLink(), path.path.parent().value_or(CanonPath::root))};
+        auto p = path.parent().resolveSymlinks() + path.baseName();
+        if (p.lstat().type != InputAccessor::tSymlink) break;
+        path = {path.accessor, CanonPath(p.readLink(), path.path.parent().value_or(CanonPath::root))};
     }
 
     /* If `path' refers to a directory, append `/default.nix'. */
-    if (path.lstat().type == InputAccessor::tDirectory)
+    if (path.resolveSymlinks().lstat().type == InputAccessor::tDirectory)
         return path + "default.nix";
 
     return path;
@@ -716,7 +717,7 @@ Expr * EvalState::parseExprFromFile(const SourcePath & path)
 
 Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv)
 {
-    auto buffer = path.readFile();
+    auto buffer = path.resolveSymlinks().readFile();
     // readFile hopefully have left some extra space for terminators
     buffer.append("\0\0", 2);
     return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv);
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index c442de986..f2d51f8f5 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -110,7 +110,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
     return res;
 }
 
-static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v)
+static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, bool resolveSymlinks = true)
 {
     NixStringContext context;
 
@@ -120,9 +120,9 @@ static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v)
         if (!context.empty() && path.accessor == state.rootFS) {
             auto rewrites = state.realiseContext(context);
             auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context);
-            return {path.accessor, CanonPath(realPath)};
-        } else
-            return path;
+            path = {path.accessor, CanonPath(realPath)};
+        }
+        return resolveSymlinks ? path.resolveSymlinks() : path;
     } catch (Error & e) {
         e.addTrace(state.positions[pos], "while realising the context of path '%s'", path);
         throw;
@@ -162,7 +162,7 @@ static void mkOutputString(
    argument. */
 static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * vScope, Value & v)
 {
-    auto path = realisePath(state, pos, vPath);
+    auto path = realisePath(state, pos, vPath, false);
     auto path2 = path.path.abs();
 
     // FIXME
@@ -1525,16 +1525,16 @@ static RegisterPrimOp primop_storePath({
 
 static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    auto & arg = *args[0];
-
-    auto path = realisePath(state, pos, arg);
-
-    /* SourcePath doesn't know about trailing slash. */
-    auto mustBeDir = arg.type() == nString
-        && (arg.string_view().ends_with("/")
-            || arg.string_view().ends_with("/."));
-
     try {
+        auto & arg = *args[0];
+
+        auto path = realisePath(state, pos, arg);
+
+        /* SourcePath doesn't know about trailing slash. */
+        auto mustBeDir = arg.type() == nString
+            && (arg.string_view().ends_with("/")
+                || arg.string_view().ends_with("/."));
+
         auto st = path.maybeLstat();
         auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory);
         v.mkBool(exists);
@@ -1771,7 +1771,7 @@ static std::string_view fileTypeToString(InputAccessor::Type type)
 
 static void prim_readFileType(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    auto path = realisePath(state, pos, *args[0]);
+    auto path = realisePath(state, pos, *args[0], false);
     /* Retrieve the directory entry type and stringize it. */
     v.mkString(fileTypeToString(path.lstat().type));
 }
diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc
index dc96f84e5..0601e6387 100644
--- a/src/libutil/posix-source-accessor.cc
+++ b/src/libutil/posix-source-accessor.cc
@@ -8,9 +8,9 @@ void PosixSourceAccessor::readFile(
     Sink & sink,
     std::function<void(uint64_t)> sizeCallback)
 {
-    // FIXME: add O_NOFOLLOW since symlinks should be resolved by the
-    // caller?
-    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
+    assertNoSymlinks(path);
+
+    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC | O_NOFOLLOW);
     if (!fd)
         throw SysError("opening file '%1%'", path);
 
@@ -42,14 +42,16 @@ void PosixSourceAccessor::readFile(
 
 bool PosixSourceAccessor::pathExists(const CanonPath & path)
 {
+    if (auto parent = path.parent()) assertNoSymlinks(*parent);
     return nix::pathExists(path.abs());
 }
 
 std::optional<SourceAccessor::Stat> PosixSourceAccessor::maybeLstat(const CanonPath & path)
 {
+    if (auto parent = path.parent()) assertNoSymlinks(*parent);
     struct stat st;
     if (::lstat(path.c_str(), &st)) {
-        if (errno == ENOENT) return std::nullopt;
+        if (errno == ENOENT || errno == ENOTDIR) return std::nullopt;
         throw SysError("getting status of '%s'", showPath(path));
     }
     mtime = std::max(mtime, st.st_mtime);
@@ -66,6 +68,7 @@ std::optional<SourceAccessor::Stat> PosixSourceAccessor::maybeLstat(const CanonP
 
 SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path)
 {
+    assertNoSymlinks(path);
     DirEntries res;
     for (auto & entry : nix::readDirectory(path.abs())) {
         std::optional<Type> type;
@@ -81,6 +84,7 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath &
 
 std::string PosixSourceAccessor::readLink(const CanonPath & path)
 {
+    if (auto parent = path.parent()) assertNoSymlinks(*parent);
     return nix::readLink(path.abs());
 }
 
@@ -89,4 +93,19 @@ std::optional<CanonPath> PosixSourceAccessor::getPhysicalPath(const CanonPath &
     return path;
 }
 
+void PosixSourceAccessor::assertNoSymlinks(CanonPath path)
+{
+    // FIXME: cache this since it potentially causes a lot of lstat calls.
+    while (!path.isRoot()) {
+        struct stat st;
+        if (::lstat(path.c_str(), &st)) {
+            if (errno != ENOENT)
+                throw SysError("getting status of '%s'", showPath(path));
+        }
+        if (S_ISLNK(st.st_mode))
+            throw Error("path '%s' is a symlink", showPath(path));
+        path.pop();
+    }
+}
+
 }
diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh
index a45d96bf8..7189a40e5 100644
--- a/src/libutil/posix-source-accessor.hh
+++ b/src/libutil/posix-source-accessor.hh
@@ -29,6 +29,11 @@ struct PosixSourceAccessor : virtual SourceAccessor
     std::string readLink(const CanonPath & path) override;
 
     std::optional<CanonPath> getPhysicalPath(const CanonPath & path) override;
+
+    /**
+     * Throw an error if `path` or any of its ancestors are symlinks.
+     */
+    void assertNoSymlinks(CanonPath path);
 };
 
 }
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 86126c7ad..e2bbd9775 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -97,7 +97,7 @@ static bool isNixExpr(const SourcePath & path, struct InputAccessor::Stat & st)
 {
     return
         st.type == InputAccessor::tRegular
-        || (st.type == InputAccessor::tDirectory && (path + "default.nix").pathExists());
+        || (st.type == InputAccessor::tDirectory && (path + "default.nix").resolveSymlinks().pathExists());
 }
 
 
@@ -116,11 +116,11 @@ static void getAllExprs(EvalState & state,
            are implemented using profiles). */
         if (i == "manifest.nix") continue;
 
-        SourcePath path2 = path + i;
+        auto path2 = (path + i).resolveSymlinks();
 
         InputAccessor::Stat st;
         try {
-            st = path2.resolveSymlinks().lstat();
+            st = path2.lstat();
         } catch (Error &) {
             continue; // ignore dangling symlinks in ~/.nix-defexpr
         }
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index 250224e7d..34f6bd005 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -21,7 +21,7 @@ DrvInfos queryInstalled(EvalState & state, const Path & userEnv)
     auto manifestFile = userEnv + "/manifest.nix";
     if (pathExists(manifestFile)) {
         Value v;
-        state.evalFile(state.rootPath(CanonPath(manifestFile)), v);
+        state.evalFile(state.rootPath(CanonPath(manifestFile)).resolveSymlinks(), v);
         Bindings & bindings(*state.allocBindings(0));
         getDerivations(state, v, "", bindings, elems, false);
     }
diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh
index cb83c34b1..2d6ab964b 100644
--- a/tests/functional/restricted.sh
+++ b/tests/functional/restricted.sh
@@ -40,13 +40,16 @@ nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT -
 [[ $(nix eval --raw --impure --restrict-eval -I . --expr 'builtins.readFile "${import ./simple.nix}/hello"') == 'Hello World!' ]]
 
 # Check that we can't follow a symlink outside of the allowed paths.
-mkdir -p $TEST_ROOT/tunnel.d
+mkdir -p $TEST_ROOT/tunnel.d $TEST_ROOT/foo2
 ln -sfn .. $TEST_ROOT/tunnel.d/tunnel
 echo foo > $TEST_ROOT/bar
 
 expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readFile <foo/tunnel/bar>" -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode"
 
-expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir <foo/tunnel>" -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode"
+expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir <foo/tunnel/foo2>" -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode"
+
+# Reading the parents of allowed paths should show only the ancestors of the allowed paths.
+[[ $(nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir <foo/tunnel>" -I $TEST_ROOT/tunnel.d) == '{ "tunnel.d" = "directory"; }' ]]
 
 # Check whether we can leak symlink information through directory traversal.
 traverseDir="$(pwd)/restricted-traverse-me"

From b23273f6a29c725646b3523b1c35a0ae4a84ef61 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 5 Dec 2023 18:10:37 -0500
Subject: [PATCH 164/421] Add missing `-pthread` for test support libraries

This is good in general (see how the other libraries also have long had
it, since 49fe9592a47e7819179c2de4fd6068e897e944c7) but in particular
needed to fix the NetBSD build.
---
 tests/unit/libexpr-support/local.mk  | 2 +-
 tests/unit/libstore-support/local.mk | 2 +-
 tests/unit/libutil-support/local.mk  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/unit/libexpr-support/local.mk b/tests/unit/libexpr-support/local.mk
index 28e87b8f2..12a76206a 100644
--- a/tests/unit/libexpr-support/local.mk
+++ b/tests/unit/libexpr-support/local.mk
@@ -20,4 +20,4 @@ libexpr-test-support_LIBS = \
     libstore-test-support libutil-test-support \
     libexpr libstore libutil
 
-libexpr-test-support_LDFLAGS := -lrapidcheck
+libexpr-test-support_LDFLAGS := -pthread -lrapidcheck
diff --git a/tests/unit/libstore-support/local.mk b/tests/unit/libstore-support/local.mk
index d5d657c91..ff075c96a 100644
--- a/tests/unit/libstore-support/local.mk
+++ b/tests/unit/libstore-support/local.mk
@@ -18,4 +18,4 @@ libstore-test-support_LIBS = \
     libutil-test-support \
     libstore libutil
 
-libstore-test-support_LDFLAGS := -lrapidcheck
+libstore-test-support_LDFLAGS := -pthread -lrapidcheck
diff --git a/tests/unit/libutil-support/local.mk b/tests/unit/libutil-support/local.mk
index 43a1551e5..2ee2cdb6c 100644
--- a/tests/unit/libutil-support/local.mk
+++ b/tests/unit/libutil-support/local.mk
@@ -16,4 +16,4 @@ libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES)
 
 libutil-test-support_LIBS = libutil
 
-libutil-test-support_LDFLAGS := -lrapidcheck
+libutil-test-support_LDFLAGS := -pthread -lrapidcheck

From 504e4fc4576dc6a4cd5c083a3bf7b80dfb0ca220 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Dec 2023 13:45:59 +0100
Subject: [PATCH 165/421] CanonPath: Support std::hash

---
 src/libfetchers/git-utils.cc |  2 +-
 src/libutil/canon-path.hh    | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 19eae0e1d..5f2a7a8bc 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -554,7 +554,7 @@ struct GitInputAccessor : InputAccessor
         return toHash(*git_tree_entry_id(entry));
     }
 
-    std::map<CanonPath, TreeEntry> lookupCache;
+    std::unordered_map<CanonPath, TreeEntry> lookupCache;
 
     /* Recursively look up 'path' relative to the root. */
     git_tree_entry * lookup(const CanonPath & path)
diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh
index 6d0519f4f..6aff4ec0d 100644
--- a/src/libutil/canon-path.hh
+++ b/src/libutil/canon-path.hh
@@ -205,8 +205,19 @@ public:
      * `CanonPath(this.makeRelative(x), this) == path`.
      */
     std::string makeRelative(const CanonPath & path) const;
+
+    friend class std::hash<CanonPath>;
 };
 
 std::ostream & operator << (std::ostream & stream, const CanonPath & path);
 
 }
+
+template<>
+struct std::hash<nix::CanonPath>
+{
+    std::size_t operator ()(const nix::CanonPath & s) const noexcept
+    {
+        return std::hash<std::string>{}(s.path);
+    }
+};

From 57246c4c3802920e6167fd540dae2a0abca97f15 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Dec 2023 13:55:07 +0100
Subject: [PATCH 166/421] PosixSourceAccessor: Cache lstat() calls

Since we're doing a lot of them in assertNoSymlinks().
---
 src/libutil/posix-source-accessor.cc | 56 +++++++++++++++++++---------
 src/libutil/posix-source-accessor.hh |  4 ++
 2 files changed, 42 insertions(+), 18 deletions(-)

diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc
index 0601e6387..15ff76e59 100644
--- a/src/libutil/posix-source-accessor.cc
+++ b/src/libutil/posix-source-accessor.cc
@@ -1,5 +1,8 @@
 #include "posix-source-accessor.hh"
 #include "signals.hh"
+#include "sync.hh"
+
+#include <unordered_map>
 
 namespace nix {
 
@@ -46,23 +49,45 @@ bool PosixSourceAccessor::pathExists(const CanonPath & path)
     return nix::pathExists(path.abs());
 }
 
+std::optional<struct stat> PosixSourceAccessor::cachedLstat(const CanonPath & path)
+{
+    static Sync<std::unordered_map<CanonPath, std::optional<struct stat>>> _cache;
+
+    {
+        auto cache(_cache.lock());
+        auto i = cache->find(path);
+        if (i != cache->end()) return i->second;
+    }
+
+    std::optional<struct stat> st{std::in_place};
+    if (::lstat(path.c_str(), &*st)) {
+        if (errno == ENOENT || errno == ENOTDIR)
+            st.reset();
+        else
+            throw SysError("getting status of '%s'", showPath(path));
+    }
+
+    auto cache(_cache.lock());
+    if (cache->size() >= 16384) cache->clear();
+    cache->emplace(path, st);
+
+    return st;
+}
+
 std::optional<SourceAccessor::Stat> PosixSourceAccessor::maybeLstat(const CanonPath & path)
 {
     if (auto parent = path.parent()) assertNoSymlinks(*parent);
-    struct stat st;
-    if (::lstat(path.c_str(), &st)) {
-        if (errno == ENOENT || errno == ENOTDIR) return std::nullopt;
-        throw SysError("getting status of '%s'", showPath(path));
-    }
-    mtime = std::max(mtime, st.st_mtime);
+    auto st = cachedLstat(path);
+    if (!st) return std::nullopt;
+    mtime = std::max(mtime, st->st_mtime);
     return Stat {
         .type =
-            S_ISREG(st.st_mode) ? tRegular :
-            S_ISDIR(st.st_mode) ? tDirectory :
-            S_ISLNK(st.st_mode) ? tSymlink :
+            S_ISREG(st->st_mode) ? tRegular :
+            S_ISDIR(st->st_mode) ? tDirectory :
+            S_ISLNK(st->st_mode) ? tSymlink :
             tMisc,
-        .fileSize = S_ISREG(st.st_mode) ? std::optional<uint64_t>(st.st_size) : std::nullopt,
-        .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR,
+        .fileSize = S_ISREG(st->st_mode) ? std::optional<uint64_t>(st->st_size) : std::nullopt,
+        .isExecutable = S_ISREG(st->st_mode) && st->st_mode & S_IXUSR,
     };
 }
 
@@ -95,14 +120,9 @@ std::optional<CanonPath> PosixSourceAccessor::getPhysicalPath(const CanonPath &
 
 void PosixSourceAccessor::assertNoSymlinks(CanonPath path)
 {
-    // FIXME: cache this since it potentially causes a lot of lstat calls.
     while (!path.isRoot()) {
-        struct stat st;
-        if (::lstat(path.c_str(), &st)) {
-            if (errno != ENOENT)
-                throw SysError("getting status of '%s'", showPath(path));
-        }
-        if (S_ISLNK(st.st_mode))
+        auto st = cachedLstat(path);
+        if (st && S_ISLNK(st->st_mode))
             throw Error("path '%s' is a symlink", showPath(path));
         path.pop();
     }
diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh
index 7189a40e5..b2bd39805 100644
--- a/src/libutil/posix-source-accessor.hh
+++ b/src/libutil/posix-source-accessor.hh
@@ -30,10 +30,14 @@ struct PosixSourceAccessor : virtual SourceAccessor
 
     std::optional<CanonPath> getPhysicalPath(const CanonPath & path) override;
 
+private:
+
     /**
      * Throw an error if `path` or any of its ancestors are symlinks.
      */
     void assertNoSymlinks(CanonPath path);
+
+    std::optional<struct stat> cachedLstat(const CanonPath & path);
 };
 
 }

From 53ab5d87c2eef72202bd76eb43e072636bbc72e8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Dec 2023 14:05:32 +0100
Subject: [PATCH 167/421] Use expectStderr

---
 tests/functional/restricted.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh
index 2d6ab964b..3de26eb36 100644
--- a/tests/functional/restricted.sh
+++ b/tests/functional/restricted.sh
@@ -14,7 +14,7 @@ nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I sr
 (! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel')
 nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel' -I src=../../src
 
-(! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile <foo/simple.nix>')
+expectStderr 1 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile <foo/simple.nix>' | grepQuiet "forbidden in restricted mode"
 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile <foo/simple.nix>' -I src=.
 
 p=$(nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval --allowed-uris "file://$(pwd)")

From ee8540ae9055791cfec4cbf8cb6335368b867acc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <7226587+thufschmitt@users.noreply.github.com>
Date: Wed, 6 Dec 2023 14:07:08 +0100
Subject: [PATCH 168/421] Fix the labeler.yml config file

labeler 5.0 changed the configuration file in a non-backwards-compatible way (https://github.com/actions/labeler/tree/main#breaking-changes-in-v5), so update our config file to match that (because all the CIs are red otherwise :grimacing: ).
---
 .github/labeler.yml | 31 +++++++++++++++++++------------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/.github/labeler.yml b/.github/labeler.yml
index 7544f07a6..b1b18c488 100644
--- a/.github/labeler.yml
+++ b/.github/labeler.yml
@@ -1,23 +1,30 @@
 "documentation":
-  - doc/manual/*
-  - src/nix/**/*.md
+  - changed-files:
+    - any-glob-to-any-file: "doc/manual/*"
+    - any-glob-to-any-file: "src/nix/**/*.md"
 
 "store":
-  - src/libstore/store-api.*
-  - src/libstore/*-store.*
+  - changed-files:
+    - any-glob-to-any-file: "src/libstore/store-api.*"
+    - any-glob-to-any-file: "src/libstore/*-store.*"
 
 "fetching":
-  - src/libfetchers/**/*
+  - changed-files:
+    - any-glob-to-any-file: "src/libfetchers/**/*"
 
 "repl":
-  - src/libcmd/repl.*
-  - src/nix/repl.*
+  - changed-files:
+    - any-glob-to-any-file: "src/libcmd/repl.*"
+    - any-glob-to-any-file: "src/nix/repl.*"
 
 "new-cli":
-  - src/nix/**/*
+  - changed-files:
+    - any-glob-to-any-file: "src/nix/**/*"
 
 "with-tests":
-  # Unit tests
-  - src/*/tests/**/*
-  # Functional and integration tests
-  - tests/functional/**/*
+  - changed-files:
+    # Unit tests
+    - any-glob-to-any-file: "src/*/tests/**/*"
+    # Functional and integration tests
+    - any-glob-to-any-file: "tests/functional/**/*"
+    

From 2bd83225004012af97d2d5977dc1de952f60aa8d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Dec 2023 14:08:40 +0100
Subject: [PATCH 169/421] Update src/libfetchers/filtering-input-accessor.hh

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 src/libfetchers/filtering-input-accessor.hh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libfetchers/filtering-input-accessor.hh b/src/libfetchers/filtering-input-accessor.hh
index 209d26974..e1b83c929 100644
--- a/src/libfetchers/filtering-input-accessor.hh
+++ b/src/libfetchers/filtering-input-accessor.hh
@@ -13,8 +13,8 @@ typedef std::function<RestrictedPathError(const CanonPath & path)> MakeNotAllowe
 
 /**
  * An abstract wrapping `InputAccessor` that performs access
- * control. Subclasses should override `checkAccess()` to implement an
- * access control policy.
+ * control. Subclasses should override `isAllowed()` to implement an
+ * access control policy. The error message is customized at construction.
  */
 struct FilteringInputAccessor : InputAccessor
 {

From 7fff625e39fa6b11c4c61eeacadc70a0253bdab6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <7226587+thufschmitt@users.noreply.github.com>
Date: Wed, 6 Dec 2023 14:13:45 +0100
Subject: [PATCH 170/421] =?UTF-8?q?Improve=20the=20error=20message=20for?=
 =?UTF-8?q?=20=E2=80=9Cmulticommands=E2=80=9D=20commands=20(#9510)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Factor out the default `MultiCommand` behavior

All the `MultiCommand`s had (nearly) the same behavior when called
without a subcommand.
Factor out this behavior into the `NixMultiCommand` class.

* Display the list of available subcommands when none is specified

Whenever a user runs a command that excepts a subcommand, add the list
of available subcommands to the error message.

* Print the multi-command lists as Markdown lists

This takes more screen real estate, but is also much more readable than
a comma-separated list
---
 src/libcmd/command.cc  | 14 ++++++++++++++
 src/libcmd/command.hh  |  6 +++++-
 src/libutil/args.cc    |  5 +++--
 src/libutil/args.hh    |  9 ++++++---
 src/nix/config.cc      | 11 ++---------
 src/nix/derivation.cc  | 11 ++---------
 src/nix/flake.cc       |  8 ++++----
 src/nix/hash.cc        | 11 +++--------
 src/nix/main.cc        |  2 +-
 src/nix/nar.cc         |  9 +--------
 src/nix/profile.cc     | 11 +++--------
 src/nix/realisation.cc | 11 ++---------
 src/nix/registry.cc    | 14 ++++----------
 src/nix/sigs.cc        | 11 +++--------
 src/nix/store.cc       | 11 ++---------
 15 files changed, 55 insertions(+), 89 deletions(-)

diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc
index de9f546fc..369fa6004 100644
--- a/src/libcmd/command.cc
+++ b/src/libcmd/command.cc
@@ -1,4 +1,5 @@
 #include "command.hh"
+#include "markdown.hh"
 #include "store-api.hh"
 #include "local-fs-store.hh"
 #include "derivations.hh"
@@ -34,6 +35,19 @@ nlohmann::json NixMultiCommand::toJSON()
     return MultiCommand::toJSON();
 }
 
+void NixMultiCommand::run()
+{
+    if (!command) {
+        std::set<std::string> subCommandTextLines;
+        for (auto & [name, _] : commands)
+            subCommandTextLines.insert(fmt("- `%s`", name));
+        std::string markdownError = fmt("`nix %s` requires a sub-command. Available sub-commands:\n\n%s\n",
+                commandName, concatStringsSep("\n", subCommandTextLines));
+        throw UsageError(renderMarkdownToTerminal(markdownError));
+    }
+    command->second->run();
+}
+
 StoreCommand::StoreCommand()
 {
 }
diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh
index 120c832ac..4a72627ed 100644
--- a/src/libcmd/command.hh
+++ b/src/libcmd/command.hh
@@ -26,9 +26,13 @@ static constexpr Command::Category catNixInstallation = 102;
 
 static constexpr auto installablesCategory = "Options that change the interpretation of [installables](@docroot@/command-ref/new-cli/nix.md#installables)";
 
-struct NixMultiCommand : virtual MultiCommand, virtual Command
+struct NixMultiCommand : MultiCommand, virtual Command
 {
     nlohmann::json toJSON() override;
+
+    using MultiCommand::MultiCommand;
+
+    virtual void run() override;
 };
 
 // For the overloaded run methods
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 4480a03f5..c4b2975ee 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -483,7 +483,7 @@ bool Args::processArgs(const Strings & args, bool finish)
         if (!anyCompleted)
             exp.handler.fun(ss);
 
-        /* Move the list element to the processedArgs. This is almost the same as 
+        /* Move the list element to the processedArgs. This is almost the same as
            `processedArgs.push_back(expectedArgs.front()); expectedArgs.pop_front()`,
            except that it will only adjust the next and prev pointers of the list
            elements, meaning the actual contents don't move in memory. This is
@@ -622,8 +622,9 @@ std::optional<ExperimentalFeature> Command::experimentalFeature ()
     return { Xp::NixCommand };
 }
 
-MultiCommand::MultiCommand(const Commands & commands_)
+MultiCommand::MultiCommand(std::string_view commandName, const Commands & commands_)
     : commands(commands_)
+    , commandName(commandName)
 {
     expectArgs({
         .label = "subcommand",
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 7af82b178..72278dccc 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -223,11 +223,11 @@ protected:
     std::list<ExpectedArg> expectedArgs;
     /**
      * List of processed positional argument forms.
-     * 
+     *
      * All items removed from `expectedArgs` are added here. After all
      * arguments were processed, this list should be exactly the same as
      * `expectedArgs` was before.
-     * 
+     *
      * This list is used to extend the lifetime of the argument forms.
      * If this is not done, some closures that reference the command
      * itself will segfault.
@@ -356,13 +356,16 @@ public:
      */
     std::optional<std::pair<std::string, ref<Command>>> command;
 
-    MultiCommand(const Commands & commands);
+    MultiCommand(std::string_view commandName, const Commands & commands);
 
     bool processFlag(Strings::iterator & pos, Strings::iterator end) override;
 
     bool processArgs(const Strings & args, bool finish) override;
 
     nlohmann::json toJSON() override;
+
+protected:
+    std::string commandName = "";
 };
 
 Strings argvToStrings(int argc, char * * argv);
diff --git a/src/nix/config.cc b/src/nix/config.cc
index 5b280d11d..52706afcf 100644
--- a/src/nix/config.cc
+++ b/src/nix/config.cc
@@ -7,9 +7,9 @@
 
 using namespace nix;
 
-struct CmdConfig : virtual NixMultiCommand
+struct CmdConfig : NixMultiCommand
 {
-    CmdConfig() : MultiCommand(RegisterCommand::getCommandsFor({"config"}))
+    CmdConfig() : NixMultiCommand("config", RegisterCommand::getCommandsFor({"config"}))
     { }
 
     std::string description() override
@@ -18,13 +18,6 @@ struct CmdConfig : virtual NixMultiCommand
     }
 
     Category category() override { return catUtility; }
-
-    void run() override
-    {
-        if (!command)
-            throw UsageError("'nix config' requires a sub-command.");
-        command->second->run();
-    }
 };
 
 struct CmdConfigShow : Command, MixJSON
diff --git a/src/nix/derivation.cc b/src/nix/derivation.cc
index cd3975a4f..59a78d378 100644
--- a/src/nix/derivation.cc
+++ b/src/nix/derivation.cc
@@ -2,9 +2,9 @@
 
 using namespace nix;
 
-struct CmdDerivation : virtual NixMultiCommand
+struct CmdDerivation : NixMultiCommand
 {
-    CmdDerivation() : MultiCommand(RegisterCommand::getCommandsFor({"derivation"}))
+    CmdDerivation() : NixMultiCommand("derivation", RegisterCommand::getCommandsFor({"derivation"}))
     { }
 
     std::string description() override
@@ -13,13 +13,6 @@ struct CmdDerivation : virtual NixMultiCommand
     }
 
     Category category() override { return catUtility; }
-
-    void run() override
-    {
-        if (!command)
-            throw UsageError("'nix derivation' requires a sub-command.");
-        command->second->run();
-    }
 };
 
 static auto rCmdDerivation = registerCommand<CmdDerivation>("derivation");
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index e0c67fdfa..2b6e56283 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -1399,7 +1399,9 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON
 struct CmdFlake : NixMultiCommand
 {
     CmdFlake()
-        : MultiCommand({
+        : NixMultiCommand(
+            "flake",
+            {
                 {"update", []() { return make_ref<CmdFlakeUpdate>(); }},
                 {"lock", []() { return make_ref<CmdFlakeLock>(); }},
                 {"metadata", []() { return make_ref<CmdFlakeMetadata>(); }},
@@ -1429,10 +1431,8 @@ struct CmdFlake : NixMultiCommand
 
     void run() override
     {
-        if (!command)
-            throw UsageError("'nix flake' requires a sub-command.");
         experimentalFeatureSettings.require(Xp::Flakes);
-        command->second->run();
+        NixMultiCommand::run();
     }
 };
 
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index d6595dcca..ededf6ef2 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -130,7 +130,9 @@ struct CmdToBase : Command
 struct CmdHash : NixMultiCommand
 {
     CmdHash()
-        : MultiCommand({
+        : NixMultiCommand(
+            "hash",
+            {
                 {"file", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Flat);; }},
                 {"path", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Recursive); }},
                 {"to-base16", []() { return make_ref<CmdToBase>(HashFormat::Base16); }},
@@ -146,13 +148,6 @@ struct CmdHash : NixMultiCommand
     }
 
     Category category() override { return catUtility; }
-
-    void run() override
-    {
-        if (!command)
-            throw UsageError("'nix hash' requires a sub-command.");
-        command->second->run();
-    }
 };
 
 static auto rCmdHash = registerCommand<CmdHash>("hash");
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 3d44e4a9d..109d2cc04 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -67,7 +67,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
     bool helpRequested = false;
     bool showVersion = false;
 
-    NixArgs() : MultiCommand(RegisterCommand::getCommandsFor({})), MixCommonArgs("nix")
+    NixArgs() : MultiCommand("", RegisterCommand::getCommandsFor({})), MixCommonArgs("nix")
     {
         categories.clear();
         categories[catHelp] = "Help commands";
diff --git a/src/nix/nar.cc b/src/nix/nar.cc
index 9815410cf..8ad4f92a7 100644
--- a/src/nix/nar.cc
+++ b/src/nix/nar.cc
@@ -4,7 +4,7 @@ using namespace nix;
 
 struct CmdNar : NixMultiCommand
 {
-    CmdNar() : MultiCommand(RegisterCommand::getCommandsFor({"nar"}))
+    CmdNar() : NixMultiCommand("nar", RegisterCommand::getCommandsFor({"nar"}))
     { }
 
     std::string description() override
@@ -20,13 +20,6 @@ struct CmdNar : NixMultiCommand
     }
 
     Category category() override { return catUtility; }
-
-    void run() override
-    {
-        if (!command)
-            throw UsageError("'nix nar' requires a sub-command.");
-        command->second->run();
-    }
 };
 
 static auto rCmdNar = registerCommand<CmdNar>("nar");
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 476ddcd60..147b4680b 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -825,7 +825,9 @@ struct CmdProfileWipeHistory : virtual StoreCommand, MixDefaultProfile, MixDryRu
 struct CmdProfile : NixMultiCommand
 {
     CmdProfile()
-        : MultiCommand({
+        : NixMultiCommand(
+            "profile",
+            {
               {"install", []() { return make_ref<CmdProfileInstall>(); }},
               {"remove", []() { return make_ref<CmdProfileRemove>(); }},
               {"upgrade", []() { return make_ref<CmdProfileUpgrade>(); }},
@@ -848,13 +850,6 @@ struct CmdProfile : NixMultiCommand
           #include "profile.md"
           ;
     }
-
-    void run() override
-    {
-        if (!command)
-            throw UsageError("'nix profile' requires a sub-command.");
-        command->second->run();
-    }
 };
 
 static auto rCmdProfile = registerCommand<CmdProfile>("profile");
diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc
index e19e93219..e1f231222 100644
--- a/src/nix/realisation.cc
+++ b/src/nix/realisation.cc
@@ -5,9 +5,9 @@
 
 using namespace nix;
 
-struct CmdRealisation : virtual NixMultiCommand
+struct CmdRealisation : NixMultiCommand
 {
-    CmdRealisation() : MultiCommand(RegisterCommand::getCommandsFor({"realisation"}))
+    CmdRealisation() : NixMultiCommand("realisation", RegisterCommand::getCommandsFor({"realisation"}))
     { }
 
     std::string description() override
@@ -16,13 +16,6 @@ struct CmdRealisation : virtual NixMultiCommand
     }
 
     Category category() override { return catUtility; }
-
-    void run() override
-    {
-        if (!command)
-            throw UsageError("'nix realisation' requires a sub-command.");
-        command->second->run();
-    }
 };
 
 static auto rCmdRealisation = registerCommand<CmdRealisation>("realisation");
diff --git a/src/nix/registry.cc b/src/nix/registry.cc
index f509ccae8..0346ec1e0 100644
--- a/src/nix/registry.cc
+++ b/src/nix/registry.cc
@@ -196,10 +196,12 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand
     }
 };
 
-struct CmdRegistry : virtual NixMultiCommand
+struct CmdRegistry : NixMultiCommand
 {
     CmdRegistry()
-        : MultiCommand({
+        : NixMultiCommand(
+            "registry",
+            {
                 {"list", []() { return make_ref<CmdRegistryList>(); }},
                 {"add", []() { return make_ref<CmdRegistryAdd>(); }},
                 {"remove", []() { return make_ref<CmdRegistryRemove>(); }},
@@ -221,14 +223,6 @@ struct CmdRegistry : virtual NixMultiCommand
     }
 
     Category category() override { return catSecondary; }
-
-    void run() override
-    {
-        experimentalFeatureSettings.require(Xp::Flakes);
-        if (!command)
-            throw UsageError("'nix registry' requires a sub-command.");
-        command->second->run();
-    }
 };
 
 static auto rCmdRegistry = registerCommand<CmdRegistry>("registry");
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index 39555c9ea..a57a407e6 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -205,7 +205,9 @@ struct CmdKeyConvertSecretToPublic : Command
 struct CmdKey : NixMultiCommand
 {
     CmdKey()
-        : MultiCommand({
+        : NixMultiCommand(
+            "key",
+            {
                 {"generate-secret", []() { return make_ref<CmdKeyGenerateSecret>(); }},
                 {"convert-secret-to-public", []() { return make_ref<CmdKeyConvertSecretToPublic>(); }},
             })
@@ -218,13 +220,6 @@ struct CmdKey : NixMultiCommand
     }
 
     Category category() override { return catUtility; }
-
-    void run() override
-    {
-        if (!command)
-            throw UsageError("'nix key' requires a sub-command.");
-        command->second->run();
-    }
 };
 
 static auto rCmdKey = registerCommand<CmdKey>("key");
diff --git a/src/nix/store.cc b/src/nix/store.cc
index 2879e03b3..79b41e096 100644
--- a/src/nix/store.cc
+++ b/src/nix/store.cc
@@ -2,9 +2,9 @@
 
 using namespace nix;
 
-struct CmdStore : virtual NixMultiCommand
+struct CmdStore : NixMultiCommand
 {
-    CmdStore() : MultiCommand(RegisterCommand::getCommandsFor({"store"}))
+    CmdStore() : NixMultiCommand("store", RegisterCommand::getCommandsFor({"store"}))
     { }
 
     std::string description() override
@@ -13,13 +13,6 @@ struct CmdStore : virtual NixMultiCommand
     }
 
     Category category() override { return catUtility; }
-
-    void run() override
-    {
-        if (!command)
-            throw UsageError("'nix store' requires a sub-command.");
-        command->second->run();
-    }
 };
 
 static auto rCmdStore = registerCommand<CmdStore>("store");

From e7abf60a0c8db19927e4fb195789b698c84e8d5a Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Sat, 25 Nov 2023 17:33:44 +0100
Subject: [PATCH 171/421] hash.cc/hash.h: Minor C++ improvements

---
 src/libutil/hash.hh | 14 +++++++-------
 src/nix/hash.cc     |  2 +-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index 6ade6555c..0e5c91b79 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -52,7 +52,7 @@ struct Hash
     /**
      * Create a zero-filled hash object.
      */
-    Hash(HashType type);
+    explicit Hash(HashType type);
 
     /**
      * Parse the hash from a string representation in the format
@@ -103,7 +103,7 @@ public:
     /**
      * Returns the length of a base-16 representation of this hash.
      */
-    size_t base16Len() const
+    [[nodiscard]] size_t base16Len() const
     {
         return hashSize * 2;
     }
@@ -111,7 +111,7 @@ public:
     /**
      * Returns the length of a base-32 representation of this hash.
      */
-    size_t base32Len() const
+    [[nodiscard]] size_t base32Len() const
     {
         return (hashSize * 8 - 1) / 5 + 1;
     }
@@ -119,7 +119,7 @@ public:
     /**
      * Returns the length of a base-64 representation of this hash.
      */
-    size_t base64Len() const
+    [[nodiscard]] size_t base64Len() const
     {
         return ((4 * hashSize / 3) + 3) & ~3;
     }
@@ -129,14 +129,14 @@ public:
      * or base-64. By default, this is prefixed by the hash type
      * (e.g. "sha256:").
      */
-    std::string to_string(HashFormat hashFormat, bool includeType) const;
+    [[nodiscard]] std::string to_string(HashFormat hashFormat, bool includeType) const;
 
-    std::string gitRev() const
+    [[nodiscard]] std::string gitRev() const
     {
         return to_string(HashFormat::Base16, false);
     }
 
-    std::string gitShortRev() const
+    [[nodiscard]] std::string gitShortRev() const
     {
         return std::string(to_string(HashFormat::Base16, false), 0, 7);
     }
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index ededf6ef2..cac65006b 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -17,7 +17,7 @@ struct CmdHashBase : Command
     std::vector<std::string> paths;
     std::optional<std::string> modulus;
 
-    CmdHashBase(FileIngestionMethod mode) : mode(mode)
+    explicit CmdHashBase(FileIngestionMethod mode) : mode(mode)
     {
         addFlag({
             .longName = "sri",

From 156ea78d7402368e3816855800eb6e0ed33a1ecc Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Sat, 25 Nov 2023 17:34:16 +0100
Subject: [PATCH 172/421] CmdHashBase: doc comment

---
 src/nix/hash.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index cac65006b..dfef44221 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -8,6 +8,11 @@
 
 using namespace nix;
 
+/**
+ * Base for `nix hash file` (deprecated), `nix hash path` and `nix-hash` (legacy).
+ *
+ * Deprecation Issue: https://github.com/NixOS/nix/issues/8876
+ */
 struct CmdHashBase : Command
 {
     FileIngestionMethod mode;

From 6bbd900d4f9983f74dcd9a0f85ab899331f661c7 Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Sat, 25 Nov 2023 17:35:24 +0100
Subject: [PATCH 173/421] nix hash convert: added

This deviated from the proposal! See comments on the issue.

https://github.com/NixOS/nix/issues/8876
---
 src/nix/hash.cc          | 63 ++++++++++++++++++++++++++++++++++++++++
 tests/functional/hash.sh | 19 +++++++++++-
 2 files changed, 81 insertions(+), 1 deletion(-)

diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index dfef44221..2b32ac03c 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -132,12 +132,75 @@ struct CmdToBase : Command
     }
 };
 
+/**
+ * `nix hash convert`
+ */
+struct CmdHashConvert : Command
+{
+    std::optional<HashFormat> from;
+    HashFormat to;
+    std::optional<HashType> type;
+    std::vector<std::string> hashStrings;
+
+    CmdHashConvert(): to(HashFormat::SRI) {
+        addFlag({
+            .longName = "from",
+            // TODO: List format choices. Maybe introduce a constant?
+            .description = "The format of the input hash.",
+            .labels = {"hash format"},
+            .handler = {[this](std::string str) {
+                from = parseHashFormat(str);
+            }},
+        });
+        addFlag({
+            .longName = "to",
+            // TODO: List format choices. Maybe introduce a constant?
+            .description = "The format of the output hash.",
+            .labels = {"hash format"},
+            .handler = {[this](std::string str) {
+                to = parseHashFormat(str);
+            }},
+        });
+        addFlag({
+            .longName = "type",
+            .description = "Specify the type if it can't be auto-detected.",
+            .labels = {"hash type"},
+            .handler = {[this](std::string str) {
+                type = parseHashType(str);
+            }},
+        });
+        expectArgs({
+           .label = "hashes",
+           .handler = {&hashStrings},
+        });
+    }
+
+    std::string description() override
+    {
+        return "convert between different hash formats, e.g. base16 and sri.";
+    }
+
+    Category category() override { return catUtility; }
+
+    void run() override {
+        for (const auto& s: hashStrings) {
+            Hash h = Hash::parseAny(s, type);
+            if (from && h.to_string(*from, from == HashFormat::SRI) != s) {
+                auto from_as_string = printHashFormat(*from);
+                throw BadHash("input hash '%s' does not have the expected format '--from %s'", s, from_as_string);
+            }
+            logger->cout(h.to_string(to, to == HashFormat::SRI));
+        }
+    }
+};
+
 struct CmdHash : NixMultiCommand
 {
     CmdHash()
         : NixMultiCommand(
             "hash",
             {
+                {"convert", []() { return make_ref<CmdHashConvert>();}},
                 {"file", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Flat);; }},
                 {"path", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Recursive); }},
                 {"to-base16", []() { return make_ref<CmdToBase>(HashFormat::Base16); }},
diff --git a/tests/functional/hash.sh b/tests/functional/hash.sh
index 34c1bb38a..d66b27a26 100644
--- a/tests/functional/hash.sh
+++ b/tests/functional/hash.sh
@@ -81,24 +81,41 @@ rm $TEST_ROOT/hash-path/hello
 ln -s x $TEST_ROOT/hash-path/hello
 try2 md5 "f78b733a68f5edbdf9413899339eaa4a"
 
-# Conversion.
+# Conversion with `nix hash` `nix-hash` and `nix hash convert`
 try3() {
+    # $1 = hash type
+    # $2 = expected hash in base16
+    # $3 = expected hash in base32
+    # $4 = expected hash in base64
+    h64=$(nix hash convert --type "$1" --to base64 "$2")
+    [ "$h64" = "$4" ]
     h64=$(nix-hash --type "$1" --to-base64 "$2")
     [ "$h64" = "$4" ]
+    # Deprecated experiment
     h64=$(nix hash to-base64 --type "$1" "$2")
     [ "$h64" = "$4" ]
+
+    sri=$(nix hash convert --type "$1" --to sri "$2")
+    [ "$sri" = "$1-$4" ]
     sri=$(nix-hash --type "$1" --to-sri "$2")
     [ "$sri" = "$1-$4" ]
     sri=$(nix hash to-sri --type "$1" "$2")
     [ "$sri" = "$1-$4" ]
+    h32=$(nix hash convert --type "$1" --to base32 "$2")
+    [ "$h32" = "$3" ]
     h32=$(nix-hash --type "$1" --to-base32 "$2")
     [ "$h32" = "$3" ]
     h32=$(nix hash to-base32 --type "$1" "$2")
     [ "$h32" = "$3" ]
     h16=$(nix-hash --type "$1" --to-base16 "$h32")
     [ "$h16" = "$2" ]
+
+    h16=$(nix hash convert --type "$1" --to base16 "$h64")
+    [ "$h16" = "$2" ]
     h16=$(nix hash to-base16 --type "$1" "$h64")
     [ "$h16" = "$2" ]
+    h16=$(nix hash convert --to base16 "$sri")
+    [ "$h16" = "$2" ]
     h16=$(nix hash to-base16 "$sri")
     [ "$h16" = "$2" ]
 }

From 0c2d5f7673ae0196b660c39b59941755103c23d0 Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Tue, 28 Nov 2023 11:42:52 +0100
Subject: [PATCH 174/421] nix hash convert: s/--type/--algo/ + more functional
 tests

https://github.com/NixOS/nix/issues/8876
---
 src/libutil/hash.hh      |  1 -
 src/nix/hash.cc          |  8 ++---
 tests/functional/hash.sh | 72 +++++++++++++++++++++++++++++++++++++---
 3 files changed, 71 insertions(+), 10 deletions(-)

diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index 0e5c91b79..820154e7a 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -40,7 +40,6 @@ enum struct HashFormat : int {
     SRI
 };
 
-
 struct Hash
 {
     constexpr static size_t maxHashSize = 64;
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 2b32ac03c..62f96ef1d 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -162,9 +162,9 @@ struct CmdHashConvert : Command
             }},
         });
         addFlag({
-            .longName = "type",
-            .description = "Specify the type if it can't be auto-detected.",
-            .labels = {"hash type"},
+            .longName = "algo",
+            .description = "Specify the algorithm if it can't be auto-detected.",
+            .labels = {"hash algorithm"},
             .handler = {[this](std::string str) {
                 type = parseHashType(str);
             }},
@@ -177,7 +177,7 @@ struct CmdHashConvert : Command
 
     std::string description() override
     {
-        return "convert between different hash formats, e.g. base16 and sri.";
+        return "convert between different hash formats, e.g. base16, nix32, base64 and sri.";
     }
 
     Category category() override { return catUtility; }
diff --git a/tests/functional/hash.sh b/tests/functional/hash.sh
index d66b27a26..031e33adf 100644
--- a/tests/functional/hash.sh
+++ b/tests/functional/hash.sh
@@ -83,11 +83,11 @@ try2 md5 "f78b733a68f5edbdf9413899339eaa4a"
 
 # Conversion with `nix hash` `nix-hash` and `nix hash convert`
 try3() {
-    # $1 = hash type
+    # $1 = hash algo
     # $2 = expected hash in base16
     # $3 = expected hash in base32
     # $4 = expected hash in base64
-    h64=$(nix hash convert --type "$1" --to base64 "$2")
+    h64=$(nix hash convert --algo "$1" --to base64 "$2")
     [ "$h64" = "$4" ]
     h64=$(nix-hash --type "$1" --to-base64 "$2")
     [ "$h64" = "$4" ]
@@ -95,13 +95,13 @@ try3() {
     h64=$(nix hash to-base64 --type "$1" "$2")
     [ "$h64" = "$4" ]
 
-    sri=$(nix hash convert --type "$1" --to sri "$2")
+    sri=$(nix hash convert --algo "$1" --to sri "$2")
     [ "$sri" = "$1-$4" ]
     sri=$(nix-hash --type "$1" --to-sri "$2")
     [ "$sri" = "$1-$4" ]
     sri=$(nix hash to-sri --type "$1" "$2")
     [ "$sri" = "$1-$4" ]
-    h32=$(nix hash convert --type "$1" --to base32 "$2")
+    h32=$(nix hash convert --algo "$1" --to base32 "$2")
     [ "$h32" = "$3" ]
     h32=$(nix-hash --type "$1" --to-base32 "$2")
     [ "$h32" = "$3" ]
@@ -110,7 +110,7 @@ try3() {
     h16=$(nix-hash --type "$1" --to-base16 "$h32")
     [ "$h16" = "$2" ]
 
-    h16=$(nix hash convert --type "$1" --to base16 "$h64")
+    h16=$(nix hash convert --algo "$1" --to base16 "$h64")
     [ "$h16" = "$2" ]
     h16=$(nix hash to-base16 --type "$1" "$h64")
     [ "$h16" = "$2" ]
@@ -118,7 +118,69 @@ try3() {
     [ "$h16" = "$2" ]
     h16=$(nix hash to-base16 "$sri")
     [ "$h16" = "$2" ]
+
+    #
+    # Converting from SRI
+    #
+
+    # Input hash algo auto-detected from SRI and output defaults to SRI as well.
+    sri=$(nix hash convert "$1-$4")
+    [ "$sri" = "$1-$4" ]
+
+    sri=$(nix hash convert --from sri "$1-$4")
+    [ "$sri" = "$1-$4" ]
+
+    sri=$(nix hash convert --to sri "$1-$4")
+    [ "$sri" = "$1-$4" ]
+
+    sri=$(nix hash convert --from sri --to sri "$1-$4")
+    [ "$sri" = "$1-$4" ]
+
+    sri=$(nix hash convert --to base64 "$1-$4")
+    [ "$sri" = "$4" ]
+
+    #
+    # Auto-detecting the input from algo and length.
+    #
+
+    sri=$(nix hash convert --algo "$1" "$2")
+    [ "$sri" = "$1-$4" ]
+    sri=$(nix hash convert --algo "$1" "$3")
+    [ "$sri" = "$1-$4" ]
+    sri=$(nix hash convert --algo "$1" "$4")
+    [ "$sri" = "$1-$4" ]
+
+    sri=$(nix hash convert --algo "$1" "$2")
+    [ "$sri" = "$1-$4" ]
+    sri=$(nix hash convert --algo "$1" "$3")
+    [ "$sri" = "$1-$4" ]
+    sri=$(nix hash convert --algo "$1" "$4")
+    [ "$sri" = "$1-$4" ]
+
+    #
+    # Asserting input format succeeds.
+    #
+
+    sri=$(nix hash convert --algo "$1" --from base16 "$2")
+    [ "$sri" = "$1-$4" ]
+    sri=$(nix hash convert --algo "$1" --from base32 "$3")
+    [ "$sri" = "$1-$4" ]
+    sri=$(nix hash convert --algo "$1" --from base64 "$4")
+    [ "$sri" = "$1-$4" ]
+
+    #
+    # Asserting input format fails.
+    #
+
+    fail=$(nix hash convert --algo "$1" --from base32 "$2" 2>&1 || echo "exit: $?")
+    [[ "$fail" == "error: input hash"*"exit: 1" ]]
+    fail=$(nix hash convert --algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?")
+    [[ "$fail" == "error: input hash"*"exit: 1" ]]
+    fail=$(nix hash convert --algo "$1" --from base32 "$4" 2>&1 || echo "exit: $?")
+    [[ "$fail" == "error: input hash"*"exit: 1" ]]
+
 }
+
 try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8="
 try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="
 try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ=="

From 5334c9c792a208db4d3824e88019a626ded1b65d Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Tue, 28 Nov 2023 14:20:27 +0100
Subject: [PATCH 175/421] HashType: Rename to HashAlgorithm

To be consistent with CLI, nix API
and many other references.

As part of this, we also converted it to a scoped enum.

https://github.com/NixOS/nix/issues/8876
---
 perl/lib/Nix/Store.xs                       |  12 +-
 src/libexpr/flake/flake.cc                  |   2 +-
 src/libexpr/primops.cc                      |  36 ++---
 src/libexpr/primops/fetchMercurial.cc       |   4 +-
 src/libexpr/primops/fetchTree.cc            |   8 +-
 src/libfetchers/fetchers.cc                 |   6 +-
 src/libfetchers/git-utils.cc                |   4 +-
 src/libfetchers/git.cc                      |   6 +-
 src/libfetchers/github.cc                   |  10 +-
 src/libfetchers/indirect.cc                 |   4 +-
 src/libfetchers/input-accessor.cc           |   4 +-
 src/libfetchers/mercurial.cc                |  10 +-
 src/libfetchers/tarball.cc                  |   6 +-
 src/libstore/binary-cache-store.cc          |  24 +--
 src/libstore/binary-cache-store.hh          |  16 +-
 src/libstore/build/local-derivation-goal.cc |  42 +++---
 src/libstore/build/worker.cc                |   4 +-
 src/libstore/builtins/fetchurl.cc           |   4 +-
 src/libstore/content-address.cc             |  28 ++--
 src/libstore/content-address.hh             |   4 +-
 src/libstore/daemon.cc                      |  16 +-
 src/libstore/derivations.cc                 |  64 ++++----
 src/libstore/derivations.hh                 |   8 +-
 src/libstore/downstream-placeholder.cc      |   4 +-
 src/libstore/export-import.cc               |   6 +-
 src/libstore/gc.cc                          |   2 +-
 src/libstore/legacy-ssh-store.cc            |  14 +-
 src/libstore/local-store.cc                 |  30 ++--
 src/libstore/local-store.hh                 |   6 +-
 src/libstore/make-content-addressed.cc      |   4 +-
 src/libstore/nar-info.cc                    |   4 +-
 src/libstore/optimise-store.cc              |   4 +-
 src/libstore/path-references.cc             |   2 +-
 src/libstore/path.cc                        |   2 +-
 src/libstore/remote-store.cc                |  28 ++--
 src/libstore/remote-store.hh                |  14 +-
 src/libstore/store-api.cc                   |  46 +++---
 src/libstore/store-api.hh                   |  22 +--
 src/libstore/store-dir-config.hh            |   2 +-
 src/libstore/worker-protocol.cc             |   2 +-
 src/libutil/args.cc                         |  14 +-
 src/libutil/args.hh                         |   6 +-
 src/libutil/git.cc                          |   8 +-
 src/libutil/git.hh                          |   6 +-
 src/libutil/hash.cc                         | 158 ++++++++++----------
 src/libutil/hash.hh                         |  38 ++---
 src/libutil/references.cc                   |   4 +-
 src/libutil/references.hh                   |   2 +-
 src/libutil/source-accessor.cc              |   8 +-
 src/libutil/source-accessor.hh              |   6 +-
 src/nix-store/nix-store.cc                  |  12 +-
 src/nix/add-to-store.cc                     |   4 +-
 src/nix/hash.cc                             |  24 +--
 src/nix/prefetch.cc                         |  34 ++---
 src/nix/profile.cc                          |   2 +-
 src/nix/verify.cc                           |   2 +-
 tests/unit/libstore/common-protocol.cc      |   8 +-
 tests/unit/libstore/derivation.cc           |   4 +-
 tests/unit/libstore/nar-info.cc             |   2 +-
 tests/unit/libstore/path-info.cc            |   2 +-
 tests/unit/libstore/serve-protocol.cc       |   8 +-
 tests/unit/libstore/worker-protocol.cc      |  10 +-
 tests/unit/libutil/git.cc                   |   8 +-
 tests/unit/libutil/hash.cc                  |  16 +-
 64 files changed, 450 insertions(+), 450 deletions(-)

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index 40257ed74..50148141b 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -205,7 +205,7 @@ void importPaths(int fd, int dontCheckSigs)
 SV * hashPath(char * algo, int base32, char * path)
     PPCODE:
         try {
-            Hash h = hashPath(parseHashType(algo), path).first;
+            Hash h = hashPath(parseHashAlgo(algo), path).first;
             auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
@@ -216,7 +216,7 @@ SV * hashPath(char * algo, int base32, char * path)
 SV * hashFile(char * algo, int base32, char * path)
     PPCODE:
         try {
-            Hash h = hashFile(parseHashType(algo), path);
+            Hash h = hashFile(parseHashAlgo(algo), path);
             auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
@@ -227,7 +227,7 @@ SV * hashFile(char * algo, int base32, char * path)
 SV * hashString(char * algo, int base32, char * s)
     PPCODE:
         try {
-            Hash h = hashString(parseHashType(algo), s);
+            Hash h = hashString(parseHashAlgo(algo), s);
             auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
@@ -238,7 +238,7 @@ SV * hashString(char * algo, int base32, char * s)
 SV * convertHash(char * algo, char * s, int toBase32)
     PPCODE:
         try {
-            auto h = Hash::parseAny(s, parseHashType(algo));
+            auto h = Hash::parseAny(s, parseHashAlgo(algo));
             auto s = h.to_string(toBase32 ? HashFormat::Base32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
@@ -281,7 +281,7 @@ SV * addToStore(char * srcPath, int recursive, char * algo)
     PPCODE:
         try {
             auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-            auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashType(algo));
+            auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashAlgo(algo));
             XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -291,7 +291,7 @@ SV * addToStore(char * srcPath, int recursive, char * algo)
 SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
     PPCODE:
         try {
-            auto h = Hash::parseAny(hash, parseHashType(algo));
+            auto h = Hash::parseAny(hash, parseHashAlgo(algo));
             auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
             auto path = store()->makeFixedOutputPath(name, FixedOutputInfo {
                 .method = method,
diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 8f8fc64f0..fee58792b 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -904,7 +904,7 @@ Fingerprint LockedFlake::getFingerprint() const
     // FIXME: as an optimization, if the flake contains a lock file
     // and we haven't changed it, then it's sufficient to use
     // flake.sourceInfo.storePath for the fingerprint.
-    return hashString(htSHA256,
+    return hashString(HashAlgorithm::SHA256,
         fmt("%s;%s;%d;%d;%s",
             flake.storePath.to_string(),
             flake.lockedRef.subdir,
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index c2499bdae..7831f3803 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1317,7 +1317,7 @@ drvName, Bindings * attrs, Value & v)
                 .errPos = state.positions[noPos]
             }));
 
-        auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo));
+        auto h = newHashAllowEmpty(*outputHash, parseHashAlgoOpt(outputHashAlgo));
 
         auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
 
@@ -1339,7 +1339,7 @@ drvName, Bindings * attrs, Value & v)
                 .errPos = state.positions[noPos]
             });
 
-        auto ht = parseHashTypeOpt(outputHashAlgo).value_or(htSHA256);
+        auto ht = parseHashAlgoOpt(outputHashAlgo).value_or(HashAlgorithm::SHA256);
         auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive);
 
         for (auto & i : outputs) {
@@ -1348,13 +1348,13 @@ drvName, Bindings * attrs, Value & v)
                 drv.outputs.insert_or_assign(i,
                     DerivationOutput::Impure {
                         .method = method,
-                        .hashType = ht,
+                        .hashAlgo = ht,
                     });
             else
                 drv.outputs.insert_or_assign(i,
                     DerivationOutput::CAFloating {
                         .method = method,
-                        .hashType = ht,
+                        .hashAlgo = ht,
                     });
         }
     }
@@ -1754,17 +1754,17 @@ static RegisterPrimOp primop_findFile(PrimOp {
 /* Return the cryptographic hash of a file in base-16. */
 static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    auto type = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashFile");
-    std::optional<HashType> ht = parseHashType(type);
-    if (!ht)
+    auto algo = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashFile");
+    std::optional<HashAlgorithm> ha = parseHashAlgo(algo);
+    if (!ha)
         state.debugThrowLastTrace(Error({
-            .msg = hintfmt("unknown hash type '%1%'", type),
+            .msg = hintfmt("unknown hash algo '%1%'", algo),
             .errPos = state.positions[pos]
         }));
 
     auto path = realisePath(state, pos, *args[1]);
 
-    v.mkString(hashString(*ht, path.readFile()).to_string(HashFormat::Base16, false));
+    v.mkString(hashString(*ha, path.readFile()).to_string(HashFormat::Base16, false));
 }
 
 static RegisterPrimOp primop_hashFile({
@@ -2341,7 +2341,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value
         else if (n == "recursive")
             method = FileIngestionMethod { state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path") };
         else if (n == "sha256")
-            expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), htSHA256);
+            expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256);
         else
             state.debugThrowLastTrace(EvalError({
                 .msg = hintfmt("unsupported argument '%1%' to 'addPath'", state.symbols[attr.name]),
@@ -3766,18 +3766,18 @@ static RegisterPrimOp primop_stringLength({
 /* Return the cryptographic hash of a string in base-16. */
 static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
-    auto type = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashString");
-    std::optional<HashType> ht = parseHashType(type);
-    if (!ht)
+    auto algo = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashString");
+    std::optional<HashAlgorithm> ha = parseHashAlgo(algo);
+    if (!ha)
         state.debugThrowLastTrace(Error({
-            .msg = hintfmt("unknown hash type '%1%'", type),
+            .msg = hintfmt("unknown hash algo '%1%'", algo),
             .errPos = state.positions[pos]
         }));
 
     NixStringContext context; // discarded
     auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString");
 
-    v.mkString(hashString(*ht, s).to_string(HashFormat::Base16, false));
+    v.mkString(hashString(*ha, s).to_string(HashFormat::Base16, false));
 }
 
 static RegisterPrimOp primop_hashString({
@@ -3800,15 +3800,15 @@ static void prim_convertHash(EvalState & state, const PosIdx pos, Value * * args
     auto hash = state.forceStringNoCtx(*iteratorHash->value, pos, "while evaluating the attribute 'hash'");
 
     Bindings::iterator iteratorHashAlgo = inputAttrs->find(state.symbols.create("hashAlgo"));
-    std::optional<HashType> ht = std::nullopt;
+    std::optional<HashAlgorithm> ha = std::nullopt;
     if (iteratorHashAlgo != inputAttrs->end()) {
-        ht = parseHashType(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'"));
+        ha = parseHashAlgo(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'"));
     }
 
     Bindings::iterator iteratorToHashFormat = getAttr(state, state.symbols.create("toHashFormat"), args[0]->attrs, "while locating the attribute 'toHashFormat'");
     HashFormat hf = parseHashFormat(state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'"));
 
-    v.mkString(Hash::parseAny(hash, ht).to_string(hf, hf == HashFormat::SRI));
+    v.mkString(Hash::parseAny(hash, ha).to_string(hf, hf == HashFormat::SRI));
 }
 
 static RegisterPrimOp primop_convertHash({
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index e76ce455d..58fe6f173 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -31,7 +31,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
                 // be both a revision or a branch/tag name.
                 auto value = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `rev` attribute passed to builtins.fetchMercurial");
                 if (std::regex_match(value.begin(), value.end(), revRegex))
-                    rev = Hash::parseAny(value, htSHA1);
+                    rev = Hash::parseAny(value, HashAlgorithm::SHA1);
                 else
                     ref = value;
             }
@@ -79,7 +79,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
         attrs2.alloc("branch").mkString(*input2.getRef());
     // Backward compatibility: set 'rev' to
     // 0000000000000000000000000000000000000000 for a dirty tree.
-    auto rev2 = input2.getRev().value_or(Hash(htSHA1));
+    auto rev2 = input2.getRev().value_or(Hash(HashAlgorithm::SHA1));
     attrs2.alloc("rev").mkString(rev2.gitRev());
     attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12));
     if (auto revCount = input2.getRevCount())
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 383ec7c58..ef80c634f 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -46,7 +46,7 @@ void emitTreeAttrs(
             attrs.alloc("shortRev").mkString(rev->gitShortRev());
         } else if (emptyRevFallback) {
             // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
-            auto emptyHash = Hash(htSHA1);
+            auto emptyHash = Hash(HashAlgorithm::SHA1);
             attrs.alloc("rev").mkString(emptyHash.gitRev());
             attrs.alloc("shortRev").mkString(emptyHash.gitShortRev());
         }
@@ -246,7 +246,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
             if (n == "url")
                 url = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the url we should fetch");
             else if (n == "sha256")
-                expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), htSHA256);
+                expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), HashAlgorithm::SHA256);
             else if (n == "name")
                 name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the name of the content we should fetch");
             else
@@ -276,7 +276,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
         state.debugThrowLastTrace(EvalError("in pure evaluation mode, '%s' requires a 'sha256' argument", who));
 
     // early exit if pinned and already in the store
-    if (expectedHash && expectedHash->type == htSHA256) {
+    if (expectedHash && expectedHash->algo == HashAlgorithm::SHA256) {
         auto expectedPath = state.store->makeFixedOutputPath(
             name,
             FixedOutputInfo {
@@ -301,7 +301,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
     if (expectedHash) {
         auto hash = unpack
             ? state.store->queryPathInfo(storePath)->narHash
-            : hashFile(htSHA256, state.store->toRealPath(storePath));
+            : hashFile(HashAlgorithm::SHA256, state.store->toRealPath(storePath));
         if (hash != *expectedHash)
             state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n  specified: %s\n  got:       %s",
                 *url, expectedHash->to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true)));
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 60208619e..573341a3d 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -289,8 +289,8 @@ std::string Input::getType() const
 std::optional<Hash> Input::getNarHash() const
 {
     if (auto s = maybeGetStrAttr(attrs, "narHash")) {
-        auto hash = s->empty() ? Hash(htSHA256) : Hash::parseSRI(*s);
-        if (hash.type != htSHA256)
+        auto hash = s->empty() ? Hash(HashAlgorithm::SHA256) : Hash::parseSRI(*s);
+        if (hash.algo != HashAlgorithm::SHA256)
             throw UsageError("narHash must use SHA-256");
         return hash;
     }
@@ -314,7 +314,7 @@ std::optional<Hash> Input::getRev() const
         } catch (BadHash &e) {
             // Default to sha1 for backwards compatibility with existing
             // usages (e.g. `builtins.fetchTree` calls or flake inputs).
-            hash = Hash::parseAny(*s, htSHA1);
+            hash = Hash::parseAny(*s, HashAlgorithm::SHA1);
         }
     }
 
diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 19eae0e1d..9356e5817 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -91,7 +91,7 @@ Hash toHash(const git_oid & oid)
     #ifdef GIT_EXPERIMENTAL_SHA256
     assert(oid.type == GIT_OID_SHA1);
     #endif
-    Hash hash(htSHA1);
+    Hash hash(HashAlgorithm::SHA1);
     memcpy(hash.hash, oid.id, hash.hashSize);
     return hash;
 }
@@ -439,7 +439,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         std::string re = R"(Good "git" signature for \* with .* key SHA256:[)";
         for (const fetchers::PublicKey & k : publicKeys){
             // Calculate sha256 fingerprint from public key and escape the regex symbol '+' to match the key literally
-            auto fingerprint = trim(hashString(htSHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "=");
+            auto fingerprint = trim(hashString(HashAlgorithm::SHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "=");
             auto escaped_fingerprint = std::regex_replace(fingerprint, std::regex("\\+"), "\\+" );
             re += "(" + escaped_fingerprint + ")";
         }
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 8cd74057c..a89acc1c0 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -52,7 +52,7 @@ bool touchCacheFile(const Path & path, time_t touch_time)
 Path getCachePath(std::string_view key)
 {
     return getCacheDir() + "/nix/gitv3/" +
-        hashString(htSHA256, key).to_string(HashFormat::Base32, false);
+        hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Base32, false);
 }
 
 // Returns the name of the HEAD branch.
@@ -369,7 +369,7 @@ struct GitInputScheme : InputScheme
     {
         auto checkHashType = [&](const std::optional<Hash> & hash)
         {
-            if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256))
+            if (hash.has_value() && !(hash->algo == HashAlgorithm::SHA1 || hash->algo == HashAlgorithm::SHA256))
                 throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true));
         };
 
@@ -559,7 +559,7 @@ struct GitInputScheme : InputScheme
                         repoInfo.url
                         );
             } else
-                input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev());
+                input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), HashAlgorithm::SHA1).gitRev());
 
             // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder
         }
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 661ad4884..70acb9354 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -42,7 +42,7 @@ struct GitArchiveInputScheme : InputScheme
         auto size = path.size();
         if (size == 3) {
             if (std::regex_match(path[2], revRegex))
-                rev = Hash::parseAny(path[2], htSHA1);
+                rev = Hash::parseAny(path[2], HashAlgorithm::SHA1);
             else if (std::regex_match(path[2], refRegex))
                 ref = path[2];
             else
@@ -68,7 +68,7 @@ struct GitArchiveInputScheme : InputScheme
             if (name == "rev") {
                 if (rev)
                     throw BadURL("URL '%s' contains multiple commit hashes", url.url);
-                rev = Hash::parseAny(value, htSHA1);
+                rev = Hash::parseAny(value, HashAlgorithm::SHA1);
             }
             else if (name == "ref") {
                 if (!std::regex_match(value, refRegex))
@@ -284,7 +284,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
             readFile(
                 store->toRealPath(
                     downloadFile(store, url, "source", false, headers).storePath)));
-        auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1);
+        auto rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1);
         debug("HEAD revision for '%s' is %s", url, rev.gitRev());
         return rev;
     }
@@ -356,7 +356,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
             readFile(
                 store->toRealPath(
                     downloadFile(store, url, "source", false, headers).storePath)));
-        auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1);
+        auto rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1);
         debug("HEAD revision for '%s' is %s", url, rev.gitRev());
         return rev;
     }
@@ -448,7 +448,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         if(!id)
             throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref);
 
-        auto rev = Hash::parseAny(*id, htSHA1);
+        auto rev = Hash::parseAny(*id, HashAlgorithm::SHA1);
         debug("HEAD revision for '%s' is %s", fmt("%s/%s", base_url, ref), rev.gitRev());
         return rev;
     }
diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc
index 8e30284c6..002c0c292 100644
--- a/src/libfetchers/indirect.cc
+++ b/src/libfetchers/indirect.cc
@@ -20,7 +20,7 @@ struct IndirectInputScheme : InputScheme
         if (path.size() == 1) {
         } else if (path.size() == 2) {
             if (std::regex_match(path[1], revRegex))
-                rev = Hash::parseAny(path[1], htSHA1);
+                rev = Hash::parseAny(path[1], HashAlgorithm::SHA1);
             else if (std::regex_match(path[1], refRegex))
                 ref = path[1];
             else
@@ -31,7 +31,7 @@ struct IndirectInputScheme : InputScheme
             ref = path[1];
             if (!std::regex_match(path[2], revRegex))
                 throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]);
-            rev = Hash::parseAny(path[2], htSHA1);
+            rev = Hash::parseAny(path[2], HashAlgorithm::SHA1);
         } else
             throw BadURL("GitHub URL '%s' is invalid", url.url);
 
diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc
index 85dc4609f..eabef55d8 100644
--- a/src/libfetchers/input-accessor.cc
+++ b/src/libfetchers/input-accessor.cc
@@ -44,8 +44,8 @@ StorePath InputAccessor::fetchToStore(
 
     auto storePath =
         settings.readOnlyMode
-        ? store->computeStorePathFromDump(*source, name, method, htSHA256).first
-        : store->addToStoreFromDump(*source, name, method, htSHA256, repair);
+        ? store->computeStorePathFromDump(*source, name, method, HashAlgorithm::SHA256).first
+        : store->addToStoreFromDump(*source, name, method, HashAlgorithm::SHA256, repair);
 
     if (cacheKey)
         fetchers::getCache()->add(store, *cacheKey, {}, storePath, true);
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index aa991a75d..713f24bbb 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -210,7 +210,7 @@ struct MercurialInputScheme : InputScheme
                     return files.count(file);
                 };
 
-                auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter);
+                auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, filter);
 
                 return {std::move(storePath), input};
             }
@@ -220,7 +220,7 @@ struct MercurialInputScheme : InputScheme
 
         auto checkHashType = [&](const std::optional<Hash> & hash)
         {
-            if (hash.has_value() && hash->type != htSHA1)
+            if (hash.has_value() && hash->algo != HashAlgorithm::SHA1)
                 throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(HashFormat::Base16, true));
         };
 
@@ -260,14 +260,14 @@ struct MercurialInputScheme : InputScheme
         });
 
         if (auto res = getCache()->lookup(store, unlockedAttrs)) {
-            auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
+            auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), HashAlgorithm::SHA1);
             if (!input.getRev() || input.getRev() == rev2) {
                 input.attrs.insert_or_assign("rev", rev2.gitRev());
                 return makeResult(res->first, std::move(res->second));
             }
         }
 
-        Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(HashFormat::Base32, false));
+        Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(HashAlgorithm::SHA256, actualUrl).to_string(HashFormat::Base32, false));
 
         /* If this is a commit hash that we already have, we don't
            have to pull again. */
@@ -301,7 +301,7 @@ struct MercurialInputScheme : InputScheme
             runHg({ "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
         assert(tokens.size() == 3);
 
-        input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], htSHA1).gitRev());
+        input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], HashAlgorithm::SHA1).gitRev());
         auto revCount = std::stoull(tokens[1]);
         input.attrs.insert_or_assign("ref", tokens[2]);
 
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index 0062878a9..086366180 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -73,7 +73,7 @@ DownloadFileResult downloadFile(
     } else {
         StringSink sink;
         dumpString(res.data, sink);
-        auto hash = hashString(htSHA256, res.data);
+        auto hash = hashString(HashAlgorithm::SHA256, res.data);
         ValidPathInfo info {
             *store,
             name,
@@ -82,7 +82,7 @@ DownloadFileResult downloadFile(
                 .hash = hash,
                 .references = {},
             },
-            hashString(htSHA256, sink.s),
+            hashString(HashAlgorithm::SHA256, sink.s),
         };
         info.narSize = sink.s.size();
         auto source = StringSource { sink.s };
@@ -156,7 +156,7 @@ DownloadTarballResult downloadTarball(
             throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
         auto topDir = tmpDir + "/" + members.begin()->name;
         lastModified = lstat(topDir).st_mtime;
-        unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair);
+        unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, defaultPathFilter, NoRepair);
     }
 
     Attrs infoAttrs({
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index ae483c95e..f287d72a8 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -143,9 +143,9 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     /* Read the NAR simultaneously into a CompressionSink+FileSink (to
        write the compressed NAR to disk), into a HashSink (to get the
        NAR hash), and into a NarAccessor (to get the NAR listing). */
-    HashSink fileHashSink { htSHA256 };
+    HashSink fileHashSink { HashAlgorithm::SHA256 };
     std::shared_ptr<SourceAccessor> narAccessor;
-    HashSink narHashSink { htSHA256 };
+    HashSink narHashSink { HashAlgorithm::SHA256 };
     {
     FdSink fileSink(fdTemp.get());
     TeeSink teeSinkCompressed { fileSink, fileHashSink };
@@ -301,9 +301,9 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
 }
 
 StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view name,
-    FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references)
+                                               FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references)
 {
-    if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256)
+    if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256)
         unsupported("addToStoreFromDump");
     return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info {
@@ -399,13 +399,13 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
 }
 
 StorePath BinaryCacheStore::addToStore(
-    std::string_view name,
-    const Path & srcPath,
-    FileIngestionMethod method,
-    HashType hashAlgo,
-    PathFilter & filter,
-    RepairFlag repair,
-    const StorePathSet & references)
+        std::string_view name,
+        const Path & srcPath,
+        FileIngestionMethod method,
+        HashAlgorithm hashAlgo,
+        PathFilter & filter,
+        RepairFlag repair,
+        const StorePathSet & references)
 {
     /* FIXME: Make BinaryCacheStore::addToStoreCommon support
        non-recursive+sha256 so we can just use the default
@@ -448,7 +448,7 @@ StorePath BinaryCacheStore::addTextToStore(
     const StorePathSet & references,
     RepairFlag repair)
 {
-    auto textHash = hashString(htSHA256, s);
+    auto textHash = hashString(HashAlgorithm::SHA256, s);
     auto path = makeTextPath(name, TextInfo { { textHash }, references });
 
     if (!repair && isValidPath(path))
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index cea2a571f..395e1b479 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -124,16 +124,16 @@ public:
         RepairFlag repair, CheckSigsFlag checkSigs) override;
 
     StorePath addToStoreFromDump(Source & dump, std::string_view name,
-        FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override;
+                                 FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) override;
 
     StorePath addToStore(
-        std::string_view name,
-        const Path & srcPath,
-        FileIngestionMethod method,
-        HashType hashAlgo,
-        PathFilter & filter,
-        RepairFlag repair,
-        const StorePathSet & references) override;
+            std::string_view name,
+            const Path & srcPath,
+            FileIngestionMethod method,
+            HashAlgorithm hashAlgo,
+            PathFilter & filter,
+            RepairFlag repair,
+            const StorePathSet & references) override;
 
     StorePath addTextToStore(
         std::string_view name,
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 198402ff7..4c3dc1f5c 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -1066,7 +1066,7 @@ void LocalDerivationGoal::initTmpDir() {
             if (passAsFile.find(i.first) == passAsFile.end()) {
                 env[i.first] = i.second;
             } else {
-                auto hash = hashString(htSHA256, i.first);
+                auto hash = hashString(HashAlgorithm::SHA256, i.first);
                 std::string fn = ".attr-" + hash.to_string(HashFormat::Base32, false);
                 Path p = tmpDir + "/" + fn;
                 writeFile(p, rewriteStrings(i.second, inputRewrites));
@@ -1290,13 +1290,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In
     { throw Error("queryPathFromHashPart"); }
 
     StorePath addToStore(
-        std::string_view name,
-        const Path & srcPath,
-        FileIngestionMethod method,
-        HashType hashAlgo,
-        PathFilter & filter,
-        RepairFlag repair,
-        const StorePathSet & references) override
+            std::string_view name,
+            const Path & srcPath,
+            FileIngestionMethod method,
+            HashAlgorithm hashAlgo,
+            PathFilter & filter,
+            RepairFlag repair,
+            const StorePathSet & references) override
     { throw Error("addToStore"); }
 
     void addToStore(const ValidPathInfo & info, Source & narSource,
@@ -1318,12 +1318,12 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In
     }
 
     StorePath addToStoreFromDump(
-        Source & dump,
-        std::string_view name,
-        FileIngestionMethod method,
-        HashType hashAlgo,
-        RepairFlag repair,
-        const StorePathSet & references) override
+            Source & dump,
+            std::string_view name,
+            FileIngestionMethod method,
+            HashAlgorithm hashAlgo,
+            RepairFlag repair,
+            const StorePathSet & references) override
     {
         auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references);
         goal.addDependency(path);
@@ -2466,7 +2466,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
             rewriteOutput(outputRewrites);
             /* FIXME optimize and deduplicate with addToStore */
             std::string oldHashPart { scratchPath->hashPart() };
-            HashModuloSink caSink { outputHash.hashType, oldHashPart };
+            HashModuloSink caSink {outputHash.hashAlgo, oldHashPart };
             std::visit(overloaded {
                 [&](const TextIngestionMethod &) {
                     readFile(actualPath, caSink);
@@ -2511,7 +2511,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                                std::string(newInfo0.path.hashPart())}});
             }
 
-            HashResult narHashAndSize = hashPath(htSHA256, actualPath);
+            HashResult narHashAndSize = hashPath(HashAlgorithm::SHA256, actualPath);
             newInfo0.narHash = narHashAndSize.first;
             newInfo0.narSize = narHashAndSize.second;
 
@@ -2531,7 +2531,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                         std::string { scratchPath->hashPart() },
                         std::string { requiredFinalPath.hashPart() });
                 rewriteOutput(outputRewrites);
-                auto narHashAndSize = hashPath(htSHA256, actualPath);
+                auto narHashAndSize = hashPath(HashAlgorithm::SHA256, actualPath);
                 ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
                 newInfo0.narSize = narHashAndSize.second;
                 auto refs = rewriteRefs();
@@ -2546,7 +2546,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
 
                 auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating {
                     .method = dof.ca.method,
-                    .hashType = wanted.type,
+                    .hashAlgo = wanted.algo,
                 });
 
                 /* Check wanted hash */
@@ -2583,7 +2583,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
             [&](const DerivationOutput::Impure & doi) {
                 return newInfoFromCA(DerivationOutput::CAFloating {
                     .method = doi.method,
-                    .hashType = doi.hashType,
+                    .hashAlgo = doi.hashAlgo,
                 });
             },
 
@@ -2945,7 +2945,7 @@ StorePath LocalDerivationGoal::makeFallbackPath(OutputNameView outputName)
 {
     return worker.store.makeStorePath(
         "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),
-        Hash(htSHA256), outputPathName(drv->name, outputName));
+        Hash(HashAlgorithm::SHA256), outputPathName(drv->name, outputName));
 }
 
 
@@ -2953,7 +2953,7 @@ StorePath LocalDerivationGoal::makeFallbackPath(const StorePath & path)
 {
     return worker.store.makeStorePath(
         "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()),
-        Hash(htSHA256), path.name());
+        Hash(HashAlgorithm::SHA256), path.name());
 }
 
 
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 01f52e7ab..9b8c36286 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -519,8 +519,8 @@ bool Worker::pathContentsGood(const StorePath & path)
     if (!pathExists(store.printStorePath(path)))
         res = false;
     else {
-        HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
-        Hash nullHash(htSHA256);
+        HashResult current = hashPath(info->narHash.algo, store.printStorePath(path));
+        Hash nullHash(HashAlgorithm::SHA256);
         res = info->narHash == nullHash || info->narHash == current.first;
     }
     pathContentsGoodCache.insert_or_assign(path, res);
diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index 357800333..2086bd0b9 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -63,9 +63,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
         for (auto hashedMirror : settings.hashedMirrors.get())
             try {
                 if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
-                std::optional<HashType> ht = parseHashTypeOpt(getAttr("outputHashAlgo"));
+                std::optional<HashAlgorithm> ht = parseHashAlgoOpt(getAttr("outputHashAlgo"));
                 Hash h = newHashAllowEmpty(getAttr("outputHash"), ht);
-                fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(HashFormat::Base16, false));
+                fetch(hashedMirror + printHashAlgo(h.algo) + "/" + h.to_string(HashFormat::Base16, false));
                 return;
             } catch (Error & e) {
                 debug(e.what());
diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc
index a5f7cdf81..de8194f73 100644
--- a/src/libstore/content-address.cc
+++ b/src/libstore/content-address.cc
@@ -38,14 +38,14 @@ ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m)
     return FileIngestionMethod::Flat;
 }
 
-std::string ContentAddressMethod::render(HashType ht) const
+std::string ContentAddressMethod::render(HashAlgorithm ha) const
 {
     return std::visit(overloaded {
         [&](const TextIngestionMethod & th) {
-            return std::string{"text:"} + printHashType(ht);
+            return std::string{"text:"} + printHashAlgo(ha);
         },
         [&](const FileIngestionMethod & fim) {
-            return "fixed:" + makeFileIngestionPrefix(fim) + printHashType(ht);
+            return "fixed:" + makeFileIngestionPrefix(fim) + printHashAlgo(ha);
         }
     }, raw);
 }
@@ -67,7 +67,7 @@ std::string ContentAddress::render() const
 /**
  * Parses content address strings up to the hash.
  */
-static std::pair<ContentAddressMethod, HashType> parseContentAddressMethodPrefix(std::string_view & rest)
+static std::pair<ContentAddressMethod, HashAlgorithm> parseContentAddressMethodPrefix(std::string_view & rest)
 {
     std::string_view wholeInput { rest };
 
@@ -83,27 +83,27 @@ static std::pair<ContentAddressMethod, HashType> parseContentAddressMethodPrefix
         auto hashTypeRaw = splitPrefixTo(rest, ':');
         if (!hashTypeRaw)
             throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", wholeInput);
-        HashType hashType = parseHashType(*hashTypeRaw);
-        return hashType;
+        HashAlgorithm hashAlgo = parseHashAlgo(*hashTypeRaw);
+        return hashAlgo;
     };
 
     // Switch on prefix
     if (prefix == "text") {
         // No parsing of the ingestion method, "text" only support flat.
-        HashType hashType = parseHashType_();
+        HashAlgorithm hashAlgo = parseHashType_();
         return {
             TextIngestionMethod {},
-            std::move(hashType),
+            std::move(hashAlgo),
         };
     } else if (prefix == "fixed") {
         // Parse method
         auto method = FileIngestionMethod::Flat;
         if (splitPrefix(rest, "r:"))
             method = FileIngestionMethod::Recursive;
-        HashType hashType = parseHashType_();
+        HashAlgorithm hashAlgo = parseHashType_();
         return {
             std::move(method),
-            std::move(hashType),
+            std::move(hashAlgo),
         };
     } else
         throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
@@ -113,15 +113,15 @@ ContentAddress ContentAddress::parse(std::string_view rawCa)
 {
     auto rest = rawCa;
 
-    auto [caMethod, hashType] = parseContentAddressMethodPrefix(rest);
+    auto [caMethod, hashAlgo] = parseContentAddressMethodPrefix(rest);
 
     return ContentAddress {
         .method = std::move(caMethod),
-        .hash = Hash::parseNonSRIUnprefixed(rest, hashType),
+        .hash = Hash::parseNonSRIUnprefixed(rest, hashAlgo),
     };
 }
 
-std::pair<ContentAddressMethod, HashType> ContentAddressMethod::parse(std::string_view caMethod)
+std::pair<ContentAddressMethod, HashAlgorithm> ContentAddressMethod::parse(std::string_view caMethod)
 {
     std::string asPrefix = std::string{caMethod} + ":";
     // parseContentAddressMethodPrefix takes its argument by reference
@@ -144,7 +144,7 @@ std::string renderContentAddress(std::optional<ContentAddress> ca)
 std::string ContentAddress::printMethodAlgo() const
 {
     return method.renderPrefix()
-        + printHashType(hash.type);
+        + printHashAlgo(hash.algo);
 }
 
 bool StoreReferences::empty() const
diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh
index bdb558907..05234da38 100644
--- a/src/libstore/content-address.hh
+++ b/src/libstore/content-address.hh
@@ -94,7 +94,7 @@ struct ContentAddressMethod
     /**
      * Parse a content addressing method and hash type.
      */
-    static std::pair<ContentAddressMethod, HashType> parse(std::string_view rawCaMethod);
+    static std::pair<ContentAddressMethod, HashAlgorithm> parse(std::string_view rawCaMethod);
 
     /**
      * Render a content addressing method and hash type in a
@@ -102,7 +102,7 @@ struct ContentAddressMethod
      *
      * The rough inverse of `parse()`.
      */
-    std::string render(HashType ht) const;
+    std::string render(HashAlgorithm ha) const;
 };
 
 
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index be9b0b0d3..530b1a178 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -400,22 +400,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             logger->startWork();
             auto pathInfo = [&]() {
                 // NB: FramedSource must be out of scope before logger->stopWork();
-                auto [contentAddressMethod, hashType_] = ContentAddressMethod::parse(camStr);
-                auto hashType = hashType_; // work around clang bug
+                auto [contentAddressMethod, hashAlgo_] = ContentAddressMethod::parse(camStr);
+                auto hashAlgo = hashAlgo_; // work around clang bug
                 FramedSource source(from);
                 // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
                 return std::visit(overloaded {
                     [&](const TextIngestionMethod &) {
-                        if (hashType != htSHA256)
+                        if (hashAlgo != HashAlgorithm::SHA256)
                             throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given",
-                                name, printHashType(hashType));
+                                name, printHashAlgo(hashAlgo));
                         // We could stream this by changing Store
                         std::string contents = source.drain();
                         auto path = store->addTextToStore(name, contents, refs, repair);
                         return store->queryPathInfo(path);
                     },
                     [&](const FileIngestionMethod & fim) {
-                        auto path = store->addToStoreFromDump(source, name, fim, hashType, repair, refs);
+                        auto path = store->addToStoreFromDump(source, name, fim, hashAlgo, repair, refs);
                         return store->queryPathInfo(path);
                     },
                 }, contentAddressMethod.raw);
@@ -424,7 +424,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
             WorkerProto::Serialise<ValidPathInfo>::write(*store, wconn, *pathInfo);
         } else {
-            HashType hashAlgo;
+            HashAlgorithm hashAlgo;
             std::string baseName;
             FileIngestionMethod method;
             {
@@ -440,7 +440,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                     hashAlgoRaw = "sha256";
                     method = FileIngestionMethod::Recursive;
                 }
-                hashAlgo = parseHashType(hashAlgoRaw);
+                hashAlgo = parseHashAlgo(hashAlgoRaw);
             }
 
             auto dumpSource = sinkToSource([&](Sink & saved) {
@@ -883,7 +883,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         bool repair, dontCheckSigs;
         auto path = store->parseStorePath(readString(from));
         auto deriver = readString(from);
-        auto narHash = Hash::parseAny(readString(from), htSHA256);
+        auto narHash = Hash::parseAny(readString(from), HashAlgorithm::SHA256);
         ValidPathInfo info { path, narHash };
         if (deriver != "")
             info.deriver = store->parseStorePath(deriver);
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index dd87203b8..c68631c1a 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -215,25 +215,25 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
 
 static DerivationOutput parseDerivationOutput(
     const StoreDirConfig & store,
-    std::string_view pathS, std::string_view hashAlgo, std::string_view hashS,
+    std::string_view pathS, std::string_view hashAlgoStr, std::string_view hashS,
     const ExperimentalFeatureSettings & xpSettings)
 {
-    if (hashAlgo != "") {
-        ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgo);
+    if (hashAlgoStr != "") {
+        ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgoStr);
         if (method == TextIngestionMethod {})
             xpSettings.require(Xp::DynamicDerivations);
-        const auto hashType = parseHashType(hashAlgo);
+        const auto hashAlgo = parseHashAlgo(hashAlgoStr);
         if (hashS == "impure") {
             xpSettings.require(Xp::ImpureDerivations);
             if (pathS != "")
                 throw FormatError("impure derivation output should not specify output path");
             return DerivationOutput::Impure {
                 .method = std::move(method),
-                .hashType = std::move(hashType),
+                .hashAlgo = std::move(hashAlgo),
             };
         } else if (hashS != "") {
             validatePath(pathS);
-            auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType);
+            auto hash = Hash::parseNonSRIUnprefixed(hashS, hashAlgo);
             return DerivationOutput::CAFixed {
                 .ca = ContentAddress {
                     .method = std::move(method),
@@ -246,7 +246,7 @@ static DerivationOutput parseDerivationOutput(
                 throw FormatError("content-addressed derivation output should not specify output path");
             return DerivationOutput::CAFloating {
                 .method = std::move(method),
-                .hashType = std::move(hashType),
+                .hashAlgo = std::move(hashAlgo),
             };
         }
     } else {
@@ -547,7 +547,7 @@ std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs,
             },
             [&](const DerivationOutput::CAFloating & dof) {
                 s += ','; printUnquotedString(s, "");
-                s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashType(dof.hashType));
+                s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo));
                 s += ','; printUnquotedString(s, "");
             },
             [&](const DerivationOutput::Deferred &) {
@@ -558,7 +558,7 @@ std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs,
             [&](const DerivationOutput::Impure & doi) {
                 // FIXME
                 s += ','; printUnquotedString(s, "");
-                s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashType(doi.hashType));
+                s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo));
                 s += ','; printUnquotedString(s, "impure");
             }
         }, i.second.raw);
@@ -631,7 +631,7 @@ DerivationType BasicDerivation::type() const
         floatingCAOutputs,
         deferredIAOutputs,
         impureOutputs;
-    std::optional<HashType> floatingHashType;
+    std::optional<HashAlgorithm> floatingHashAlgo;
 
     for (auto & i : outputs) {
         std::visit(overloaded {
@@ -643,10 +643,10 @@ DerivationType BasicDerivation::type() const
             },
             [&](const DerivationOutput::CAFloating & dof) {
                 floatingCAOutputs.insert(i.first);
-                if (!floatingHashType) {
-                    floatingHashType = dof.hashType;
+                if (!floatingHashAlgo) {
+                    floatingHashAlgo = dof.hashAlgo;
                 } else {
-                    if (*floatingHashType != dof.hashType)
+                    if (*floatingHashAlgo != dof.hashAlgo)
                         throw Error("all floating outputs must use the same hash type");
                 }
             },
@@ -774,7 +774,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
         std::map<std::string, Hash> outputHashes;
         for (const auto & i : drv.outputs) {
             auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw);
-            auto hash = hashString(htSHA256, "fixed:out:"
+            auto hash = hashString(HashAlgorithm::SHA256, "fixed:out:"
                 + dof.ca.printMethodAlgo() + ":"
                 + dof.ca.hash.to_string(HashFormat::Base16, false) + ":"
                 + store.printStorePath(dof.path(store, drv.name, i.first)));
@@ -825,7 +825,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
         }
     }
 
-    auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
+    auto hash = hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2));
 
     std::map<std::string, Hash> outputHashes;
     for (const auto & [outputName, _] : drv.outputs) {
@@ -930,7 +930,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva
             },
             [&](const DerivationOutput::CAFloating & dof) {
                 out << ""
-                    << (dof.method.renderPrefix() + printHashType(dof.hashType))
+                    << (dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo))
                     << "";
             },
             [&](const DerivationOutput::Deferred &) {
@@ -940,7 +940,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva
             },
             [&](const DerivationOutput::Impure & doi) {
                 out << ""
-                    << (doi.method.renderPrefix() + printHashType(doi.hashType))
+                    << (doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo))
                     << "impure";
             },
         }, i.second.raw);
@@ -958,7 +958,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva
 std::string hashPlaceholder(const OutputNameView outputName)
 {
     // FIXME: memoize?
-    return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Base32, false);
+    return "/" + hashString(HashAlgorithm::SHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Base32, false);
 }
 
 
@@ -1150,7 +1150,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
 }
 
 
-const Hash impureOutputHash = hashString(htSHA256, "impure");
+const Hash impureOutputHash = hashString(HashAlgorithm::SHA256, "impure");
 
 nlohmann::json DerivationOutput::toJSON(
     const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const
@@ -1167,11 +1167,11 @@ nlohmann::json DerivationOutput::toJSON(
             // FIXME print refs?
         },
         [&](const DerivationOutput::CAFloating & dof) {
-            res["hashAlgo"] = dof.method.renderPrefix() + printHashType(dof.hashType);
+            res["hashAlgo"] = dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo);
         },
         [&](const DerivationOutput::Deferred &) {},
         [&](const DerivationOutput::Impure & doi) {
-            res["hashAlgo"] = doi.method.renderPrefix() + printHashType(doi.hashType);
+            res["hashAlgo"] = doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo);
             res["impure"] = true;
         },
     }, raw);
@@ -1191,15 +1191,15 @@ DerivationOutput DerivationOutput::fromJSON(
     for (const auto & [key, _] : json)
         keys.insert(key);
 
-    auto methodAlgo = [&]() -> std::pair<ContentAddressMethod, HashType> {
-        std::string hashAlgo = json["hashAlgo"];
+    auto methodAlgo = [&]() -> std::pair<ContentAddressMethod, HashAlgorithm> {
+        std::string hashAlgoStr = json["hashAlgo"];
         // remaining to parse, will be mutated by parsers
-        std::string_view s = hashAlgo;
+        std::string_view s = hashAlgoStr;
         ContentAddressMethod method = ContentAddressMethod::parsePrefix(s);
         if (method == TextIngestionMethod {})
             xpSettings.require(Xp::DynamicDerivations);
-        auto hashType = parseHashType(s);
-        return { std::move(method), std::move(hashType) };
+        auto hashAlgo = parseHashAlgo(s);
+        return { std::move(method), std::move(hashAlgo) };
     };
 
     if (keys == (std::set<std::string_view> { "path" })) {
@@ -1209,11 +1209,11 @@ DerivationOutput DerivationOutput::fromJSON(
     }
 
     else if (keys == (std::set<std::string_view> { "path", "hashAlgo", "hash" })) {
-        auto [method, hashType] = methodAlgo();
+        auto [method, hashAlgo] = methodAlgo();
         auto dof = DerivationOutput::CAFixed {
             .ca = ContentAddress {
                 .method = std::move(method),
-                .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType),
+                .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashAlgo),
             },
         };
         if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"]))
@@ -1223,10 +1223,10 @@ DerivationOutput DerivationOutput::fromJSON(
 
     else if (keys == (std::set<std::string_view> { "hashAlgo" })) {
         xpSettings.require(Xp::CaDerivations);
-        auto [method, hashType] = methodAlgo();
+        auto [method, hashAlgo] = methodAlgo();
         return DerivationOutput::CAFloating {
             .method = std::move(method),
-            .hashType = std::move(hashType),
+            .hashAlgo = std::move(hashAlgo),
         };
     }
 
@@ -1236,10 +1236,10 @@ DerivationOutput DerivationOutput::fromJSON(
 
     else if (keys == (std::set<std::string_view> { "hashAlgo", "impure" })) {
         xpSettings.require(Xp::ImpureDerivations);
-        auto [method, hashType] = methodAlgo();
+        auto [method, hashAlgo] = methodAlgo();
         return DerivationOutput::Impure {
             .method = std::move(method),
-            .hashType = hashType,
+            .hashAlgo = hashAlgo,
         };
     }
 
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 219e8e7d7..290abedcf 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -75,9 +75,9 @@ struct DerivationOutput
         /**
          * How the serialization will be hashed
          */
-        HashType hashType;
+        HashAlgorithm hashAlgo;
 
-        GENERATE_CMP(CAFloating, me->method, me->hashType);
+        GENERATE_CMP(CAFloating, me->method, me->hashAlgo);
     };
 
     /**
@@ -102,9 +102,9 @@ struct DerivationOutput
         /**
          * How the serialization will be hashed
          */
-        HashType hashType;
+        HashAlgorithm hashAlgo;
 
-        GENERATE_CMP(Impure, me->method, me->hashType);
+        GENERATE_CMP(Impure, me->method, me->hashAlgo);
     };
 
     typedef std::variant<
diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc
index ca9f7476e..10df37fa4 100644
--- a/src/libstore/downstream-placeholder.cc
+++ b/src/libstore/downstream-placeholder.cc
@@ -19,7 +19,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput(
     auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
     auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
     return DownstreamPlaceholder {
-        hashString(htSHA256, clearText)
+        hashString(HashAlgorithm::SHA256, clearText)
     };
 }
 
@@ -34,7 +34,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation(
         + compressed.to_string(HashFormat::Base32, false)
         + ":" + std::string { outputName };
     return DownstreamPlaceholder {
-        hashString(htSHA256, clearText)
+        hashString(HashAlgorithm::SHA256, clearText)
     };
 }
 
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index 52130f8f6..48718ef84 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -30,7 +30,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
 {
     auto info = queryPathInfo(path);
 
-    HashSink hashSink(htSHA256);
+    HashSink hashSink(HashAlgorithm::SHA256);
     TeeSink teeSink(sink, hashSink);
 
     narFromPath(path, teeSink);
@@ -39,7 +39,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
        filesystem corruption from spreading to other machines.
        Don't complain if the stored hash is zero (unknown). */
     Hash hash = hashSink.currentHash().first;
-    if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
+    if (hash != info->narHash && info->narHash != Hash(info->narHash.algo))
         throw Error("hash of path '%s' has changed from '%s' to '%s'!",
             printStorePath(path), info->narHash.to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true));
 
@@ -79,7 +79,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
         auto references = CommonProto::Serialise<StorePathSet>::read(*this,
             CommonProto::ReadConn { .from = source });
         auto deriver = readString(source);
-        auto narHash = hashString(htSHA256, saved.s);
+        auto narHash = hashString(HashAlgorithm::SHA256, saved.s);
 
         ValidPathInfo info { path, narHash };
         if (deriver != "")
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 93fa60682..5c413aa77 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -50,7 +50,7 @@ static void makeSymlink(const Path & link, const Path & target)
 
 void LocalStore::addIndirectRoot(const Path & path)
 {
-    std::string hash = hashString(htSHA1, path).to_string(HashFormat::Base32, false);
+    std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Base32, false);
     Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash));
     makeSymlink(realRoot, path);
 }
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 731457354..fb1580dd6 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -267,13 +267,13 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
     { unsupported("queryPathFromHashPart"); }
 
     StorePath addToStore(
-        std::string_view name,
-        const Path & srcPath,
-        FileIngestionMethod method,
-        HashType hashAlgo,
-        PathFilter & filter,
-        RepairFlag repair,
-        const StorePathSet & references) override
+            std::string_view name,
+            const Path & srcPath,
+            FileIngestionMethod method,
+            HashAlgorithm hashAlgo,
+            PathFilter & filter,
+            RepairFlag repair,
+            const StorePathSet & references) override
     { unsupported("addToStore"); }
 
     StorePath addTextToStore(
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index c8962f574..ef7dd7985 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -955,7 +955,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
         StorePathSet paths;
 
         for (auto & [_, i] : infos) {
-            assert(i.narHash.type == htSHA256);
+            assert(i.narHash.algo == HashAlgorithm::SHA256);
             if (isValidPath_(*state, i.path))
                 updatePathInfo(*state, i);
             else
@@ -1069,7 +1069,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             /* While restoring the path from the NAR, compute the hash
                of the NAR. */
-            HashSink hashSink(htSHA256);
+            HashSink hashSink(HashAlgorithm::SHA256);
 
             TeeSource wrapperSource { source, hashSink };
 
@@ -1090,7 +1090,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
                 auto & specified = *info.ca;
                 auto actualHash = hashCAPath(
                     specified.method,
-                    specified.hash.type,
+                    specified.hash.algo,
                     info.path
                 );
                 if (specified.hash != actualHash.hash) {
@@ -1116,7 +1116,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
 
 StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name,
-    FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references)
+                                         FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references)
 {
     /* For computing the store path. */
     auto hashSink = std::make_unique<HashSink>(hashAlgo);
@@ -1220,8 +1220,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
             /* For computing the nar hash. In recursive SHA-256 mode, this
                is the same as the store hash, so no need to do it again. */
             auto narHash = std::pair { hash, size };
-            if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) {
-                HashSink narSink { htSHA256 };
+            if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) {
+                HashSink narSink { HashAlgorithm::SHA256 };
                 dumpPath(realPath, narSink);
                 narHash = narSink.finish();
             }
@@ -1252,7 +1252,7 @@ StorePath LocalStore::addTextToStore(
     std::string_view s,
     const StorePathSet & references, RepairFlag repair)
 {
-    auto hash = hashString(htSHA256, s);
+    auto hash = hashString(HashAlgorithm::SHA256, s);
     auto dstPath = makeTextPath(name, TextInfo {
         .hash = hash,
         .references = references,
@@ -1278,7 +1278,7 @@ StorePath LocalStore::addTextToStore(
 
             StringSink sink;
             dumpString(s, sink);
-            auto narHash = hashString(htSHA256, sink.s);
+            auto narHash = hashString(HashAlgorithm::SHA256, sink.s);
 
             optimisePath(realPath, repair);
 
@@ -1389,7 +1389,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
         for (auto & link : readDirectory(linksDir)) {
             printMsg(lvlTalkative, "checking contents of '%s'", link.name);
             Path linkPath = linksDir + "/" + link.name;
-            std::string hash = hashPath(htSHA256, linkPath).first.to_string(HashFormat::Base32, false);
+            std::string hash = hashPath(HashAlgorithm::SHA256, linkPath).first.to_string(HashFormat::Base32, false);
             if (hash != link.name) {
                 printError("link '%s' was modified! expected hash '%s', got '%s'",
                     linkPath, link.name, hash);
@@ -1406,7 +1406,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
         printInfo("checking store hashes...");
 
-        Hash nullHash(htSHA256);
+        Hash nullHash(HashAlgorithm::SHA256);
 
         for (auto & i : validPaths) {
             try {
@@ -1415,7 +1415,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 /* Check the content hash (optionally - slow). */
                 printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i));
 
-                auto hashSink = HashSink(info->narHash.type);
+                auto hashSink = HashSink(info->narHash.algo);
 
                 dumpPath(Store::toRealPath(i), hashSink);
                 auto current = hashSink.finish();
@@ -1697,20 +1697,20 @@ void LocalStore::queryRealisationUncached(const DrvOutput & id,
 }
 
 ContentAddress LocalStore::hashCAPath(
-    const ContentAddressMethod & method, const HashType & hashType,
+    const ContentAddressMethod & method, const HashAlgorithm & hashAlgo,
     const StorePath & path)
 {
-    return hashCAPath(method, hashType, Store::toRealPath(path), path.hashPart());
+    return hashCAPath(method, hashAlgo, Store::toRealPath(path), path.hashPart());
 }
 
 ContentAddress LocalStore::hashCAPath(
     const ContentAddressMethod & method,
-    const HashType & hashType,
+    const HashAlgorithm & hashAlgo,
     const Path & path,
     const std::string_view pathHash
 )
 {
-    HashModuloSink caSink ( hashType, std::string(pathHash) );
+    HashModuloSink caSink ( hashAlgo, std::string(pathHash) );
     std::visit(overloaded {
         [&](const TextIngestionMethod &) {
             readFile(path, caSink);
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 8f0ffd2a2..ee605b5a2 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -178,7 +178,7 @@ public:
         RepairFlag repair, CheckSigsFlag checkSigs) override;
 
     StorePath addToStoreFromDump(Source & dump, std::string_view name,
-        FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override;
+                                 FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) override;
 
     StorePath addTextToStore(
         std::string_view name,
@@ -353,12 +353,12 @@ private:
     // XXX: Make a generic `Store` method
     ContentAddress hashCAPath(
         const ContentAddressMethod & method,
-        const HashType & hashType,
+        const HashAlgorithm & hashAlgo,
         const StorePath & path);
 
     ContentAddress hashCAPath(
         const ContentAddressMethod & method,
-        const HashType & hashType,
+        const HashAlgorithm & hashAlgo,
         const Path & path,
         const std::string_view pathHash
     );
diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc
index 253609ed2..170fe67b9 100644
--- a/src/libstore/make-content-addressed.cc
+++ b/src/libstore/make-content-addressed.cc
@@ -43,7 +43,7 @@ std::map<StorePath, StorePath> makeContentAddressed(
 
         sink.s = rewriteStrings(sink.s, rewrites);
 
-        HashModuloSink hashModuloSink(htSHA256, oldHashPart);
+        HashModuloSink hashModuloSink(HashAlgorithm::SHA256, oldHashPart);
         hashModuloSink(sink.s);
 
         auto narModuloHash = hashModuloSink.finish().first;
@@ -66,7 +66,7 @@ std::map<StorePath, StorePath> makeContentAddressed(
         rsink2(sink.s);
         rsink2.flush();
 
-        info.narHash = hashString(htSHA256, sink2.s);
+        info.narHash = hashString(HashAlgorithm::SHA256, sink2.s);
         info.narSize = sink.s.size();
 
         StringSource source(sink2.s);
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index 1060a6c8b..25e2a7d7b 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -113,10 +113,10 @@ std::string NarInfo::to_string(const Store & store) const
     res += "URL: " + url + "\n";
     assert(compression != "");
     res += "Compression: " + compression + "\n";
-    assert(fileHash && fileHash->type == htSHA256);
+    assert(fileHash && fileHash->algo == HashAlgorithm::SHA256);
     res += "FileHash: " + fileHash->to_string(HashFormat::Base32, true) + "\n";
     res += "FileSize: " + std::to_string(fileSize) + "\n";
-    assert(narHash.type == htSHA256);
+    assert(narHash.algo == HashAlgorithm::SHA256);
     res += "NarHash: " + narHash.to_string(HashFormat::Base32, true) + "\n";
     res += "NarSize: " + std::to_string(narSize) + "\n";
 
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 0fa977545..cadf88347 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -146,7 +146,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
        Also note that if `path' is a symlink, then we're hashing the
        contents of the symlink (i.e. the result of readlink()), not
        the contents of the target (which may not even exist). */
-    Hash hash = hashPath(htSHA256, path).first;
+    Hash hash = hashPath(HashAlgorithm::SHA256, path).first;
     debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Base32, true));
 
     /* Check if this is a known hash. */
@@ -156,7 +156,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     if (pathExists(linkPath)) {
         auto stLink = lstat(linkPath);
         if (st.st_size != stLink.st_size
-            || (repair && hash != hashPath(htSHA256, linkPath).first))
+            || (repair && hash != hashPath(HashAlgorithm::SHA256, linkPath).first))
         {
             // XXX: Consider overwriting linkPath with our valid version.
             warn("removing corrupted link '%s'", linkPath);
diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc
index 274b596c0..15f52ec9d 100644
--- a/src/libstore/path-references.cc
+++ b/src/libstore/path-references.cc
@@ -49,7 +49,7 @@ std::pair<StorePathSet, HashResult> scanForReferences(
     const std::string & path,
     const StorePathSet & refs)
 {
-    HashSink hashSink { htSHA256 };
+    HashSink hashSink { HashAlgorithm::SHA256 };
     auto found = scanForReferences(hashSink, path, refs);
     auto hash = hashSink.finish();
     return std::pair<StorePathSet, HashResult>(found, hash);
diff --git a/src/libstore/path.cc b/src/libstore/path.cc
index 69f6d7356..d5257c939 100644
--- a/src/libstore/path.cc
+++ b/src/libstore/path.cc
@@ -49,7 +49,7 @@ StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x");
 
 StorePath StorePath::random(std::string_view name)
 {
-    Hash hash(htSHA1);
+    Hash hash(HashAlgorithm::SHA1);
     randombytes_buf(hash.hash, hash.hashSize);
     return StorePath(hash, name);
 }
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 3d3919882..cc26c2a94 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -417,12 +417,12 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string &
 
 
 ref<const ValidPathInfo> RemoteStore::addCAToStore(
-    Source & dump,
-    std::string_view name,
-    ContentAddressMethod caMethod,
-    HashType hashType,
-    const StorePathSet & references,
-    RepairFlag repair)
+        Source & dump,
+        std::string_view name,
+        ContentAddressMethod caMethod,
+        HashAlgorithm hashAlgo,
+        const StorePathSet & references,
+        RepairFlag repair)
 {
     std::optional<ConnectionHandle> conn_(getConnection());
     auto & conn = *conn_;
@@ -432,7 +432,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
         conn->to
             << WorkerProto::Op::AddToStore
             << name
-            << caMethod.render(hashType);
+            << caMethod.render(hashAlgo);
         WorkerProto::write(*this, *conn, references);
         conn->to << repair;
 
@@ -453,9 +453,9 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
 
         std::visit(overloaded {
             [&](const TextIngestionMethod & thm) -> void {
-                if (hashType != htSHA256)
+                if (hashAlgo != HashAlgorithm::SHA256)
                     throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given",
-                        name, printHashType(hashType));
+                        name, printHashAlgo(hashAlgo));
                 std::string s = dump.drain();
                 conn->to << WorkerProto::Op::AddTextToStore << name << s;
                 WorkerProto::write(*this, *conn, references);
@@ -465,9 +465,9 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
                 conn->to
                     << WorkerProto::Op::AddToStore
                     << name
-                    << ((hashType == htSHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
+                    << ((hashAlgo == HashAlgorithm::SHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
                     << (fim == FileIngestionMethod::Recursive ? 1 : 0)
-                    << printHashType(hashType);
+                    << printHashAlgo(hashAlgo);
 
                 try {
                     conn->to.written = 0;
@@ -503,9 +503,9 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
 
 
 StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name,
-      FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references)
+                                          FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references)
 {
-    return addCAToStore(dump, name, method, hashType, references, repair)->path;
+    return addCAToStore(dump, name, method, hashAlgo, references, repair)->path;
 }
 
 
@@ -610,7 +610,7 @@ StorePath RemoteStore::addTextToStore(
     RepairFlag repair)
 {
     StringSource source(s);
-    return addCAToStore(source, name, TextIngestionMethod {}, htSHA256, references, repair)->path;
+    return addCAToStore(source, name, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair)->path;
 }
 
 void RemoteStore::registerDrvOutput(const Realisation & info)
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 1cc11af86..f2e34c1a3 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -74,18 +74,18 @@ public:
      * Add a content-addressable store path. `dump` will be drained.
      */
     ref<const ValidPathInfo> addCAToStore(
-        Source & dump,
-        std::string_view name,
-        ContentAddressMethod caMethod,
-        HashType hashType,
-        const StorePathSet & references,
-        RepairFlag repair);
+            Source & dump,
+            std::string_view name,
+            ContentAddressMethod caMethod,
+            HashAlgorithm hashAlgo,
+            const StorePathSet & references,
+            RepairFlag repair);
 
     /**
      * Add a content-addressable store path. Does not support references. `dump` will be drained.
      */
     StorePath addToStoreFromDump(Source & dump, std::string_view name,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override;
+                                 FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override;
 
     void addToStore(const ValidPathInfo & info, Source & nar,
         RepairFlag repair, CheckSigsFlag checkSigs) override;
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 8601e0857..800df7fa0 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -153,7 +153,7 @@ StorePath StoreDirConfig::makeStorePath(std::string_view type,
     /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
     auto s = std::string(type) + ":" + std::string(hash)
         + ":" + storeDir + ":" + std::string(name);
-    auto h = compressHash(hashString(htSHA256, s), 20);
+    auto h = compressHash(hashString(HashAlgorithm::SHA256, s), 20);
     return StorePath(h, name);
 }
 
@@ -191,12 +191,12 @@ static std::string makeType(
 
 StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
 {
-    if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) {
+    if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::Recursive) {
         return makeStorePath(makeType(*this, "source", info.references), info.hash, name);
     } else {
         assert(info.references.size() == 0);
         return makeStorePath("output:out",
-            hashString(htSHA256,
+            hashString(HashAlgorithm::SHA256,
                 "fixed:out:"
                 + makeFileIngestionPrefix(info.method)
                 + info.hash.to_string(HashFormat::Base16, true) + ":"),
@@ -207,7 +207,7 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed
 
 StorePath StoreDirConfig::makeTextPath(std::string_view name, const TextInfo & info) const
 {
-    assert(info.hash.type == htSHA256);
+    assert(info.hash.algo == HashAlgorithm::SHA256);
     return makeStorePath(
         makeType(*this, "text", StoreReferences {
             .others = info.references,
@@ -233,11 +233,11 @@ StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const
 
 
 std::pair<StorePath, Hash> StoreDirConfig::computeStorePathFromDump(
-    Source & dump,
-    std::string_view name,
-    FileIngestionMethod method,
-    HashType hashAlgo,
-    const StorePathSet & references) const
+        Source & dump,
+        std::string_view name,
+        FileIngestionMethod method,
+        HashAlgorithm hashAlgo,
+        const StorePathSet & references) const
 {
     HashSink sink(hashAlgo);
     dump.drainInto(sink);
@@ -257,20 +257,20 @@ StorePath StoreDirConfig::computeStorePathForText(
     const StorePathSet & references) const
 {
     return makeTextPath(name, TextInfo {
-        .hash = hashString(htSHA256, s),
+        .hash = hashString(HashAlgorithm::SHA256, s),
         .references = references,
     });
 }
 
 
 StorePath Store::addToStore(
-    std::string_view name,
-    const Path & _srcPath,
-    FileIngestionMethod method,
-    HashType hashAlgo,
-    PathFilter & filter,
-    RepairFlag repair,
-    const StorePathSet & references)
+        std::string_view name,
+        const Path & _srcPath,
+        FileIngestionMethod method,
+        HashAlgorithm hashAlgo,
+        PathFilter & filter,
+        RepairFlag repair,
+        const StorePathSet & references)
 {
     Path srcPath(absPath(_srcPath));
     auto source = sinkToSource([&](Sink & sink) {
@@ -405,10 +405,10 @@ digraph graphname {
 }
 */
 ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
-    FileIngestionMethod method, HashType hashAlgo,
-    std::optional<Hash> expectedCAHash)
+                                    FileIngestionMethod method, HashAlgorithm hashAlgo,
+                                    std::optional<Hash> expectedCAHash)
 {
-    HashSink narHashSink { htSHA256 };
+    HashSink narHashSink { HashAlgorithm::SHA256 };
     HashSink caHashSink { hashAlgo };
 
     /* Note that fileSink and unusualHashTee must be mutually exclusive, since
@@ -417,7 +417,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
     RegularFileSink fileSink { caHashSink };
     TeeSink unusualHashTee { narHashSink, caHashSink };
 
-    auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256
+    auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != HashAlgorithm::SHA256
         ? static_cast<Sink &>(unusualHashTee)
         : narHashSink;
 
@@ -445,7 +445,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
        finish. */
     auto [narHash, narSize] = narHashSink.finish();
 
-    auto hash = method == FileIngestionMethod::Recursive && hashAlgo == htSHA256
+    auto hash = method == FileIngestionMethod::Recursive && hashAlgo == HashAlgorithm::SHA256
         ? narHash
         : caHashSink.finish().first;
 
@@ -1205,7 +1205,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre
     if (!hashGiven) {
         std::string s;
         getline(str, s);
-        auto narHash = Hash::parseAny(s, htSHA256);
+        auto narHash = Hash::parseAny(s, HashAlgorithm::SHA256);
         getline(str, s);
         auto narSize = string2Int<uint64_t>(s);
         if (!narSize) throw Error("number expected");
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 5860d0ea6..ada6699d5 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -427,13 +427,13 @@ public:
      * libutil/archive.hh).
      */
     virtual StorePath addToStore(
-        std::string_view name,
-        const Path & srcPath,
-        FileIngestionMethod method = FileIngestionMethod::Recursive,
-        HashType hashAlgo = htSHA256,
-        PathFilter & filter = defaultPathFilter,
-        RepairFlag repair = NoRepair,
-        const StorePathSet & references = StorePathSet());
+            std::string_view name,
+            const Path & srcPath,
+            FileIngestionMethod method = FileIngestionMethod::Recursive,
+            HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
+            PathFilter & filter = defaultPathFilter,
+            RepairFlag repair = NoRepair,
+            const StorePathSet & references = StorePathSet());
 
     /**
      * Copy the contents of a path to the store and register the
@@ -441,8 +441,8 @@ public:
      * memory.
      */
     ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
-        std::optional<Hash> expectedCAHash = {});
+                                 FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
+                                 std::optional<Hash> expectedCAHash = {});
 
     /**
      * Like addToStore(), but the contents of the path are contained
@@ -454,8 +454,8 @@ public:
      * \todo remove?
      */
     virtual StorePath addToStoreFromDump(Source & dump, std::string_view name,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair,
-        const StorePathSet & references = StorePathSet())
+                                         FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, RepairFlag repair = NoRepair,
+                                         const StorePathSet & references = StorePathSet())
     { unsupported("addToStoreFromDump"); }
 
     /**
diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh
index 53843d663..8dafca096 100644
--- a/src/libstore/store-dir-config.hh
+++ b/src/libstore/store-dir-config.hh
@@ -98,7 +98,7 @@ struct StoreDirConfig : public Config
         Source & dump,
         std::string_view name,
         FileIngestionMethod method = FileIngestionMethod::Recursive,
-        HashType hashAlgo = htSHA256,
+        HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
         const StorePathSet & references = {}) const;
 
     /**
diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc
index 43654d7e8..2a379e75e 100644
--- a/src/libstore/worker-protocol.cc
+++ b/src/libstore/worker-protocol.cc
@@ -160,7 +160,7 @@ void WorkerProto::Serialise<ValidPathInfo>::write(const StoreDirConfig & store,
 UnkeyedValidPathInfo WorkerProto::Serialise<UnkeyedValidPathInfo>::read(const StoreDirConfig & store, ReadConn conn)
 {
     auto deriver = readString(conn.from);
-    auto narHash = Hash::parseAny(readString(conn.from), htSHA256);
+    auto narHash = Hash::parseAny(readString(conn.from), HashAlgorithm::SHA256);
     UnkeyedValidPathInfo info(narHash);
     if (deriver != "") info.deriver = store.parseStorePath(deriver);
     info.references = WorkerProto::Serialise<StorePathSet>::read(store, conn);
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index c4b2975ee..ac3727d11 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -546,32 +546,32 @@ nlohmann::json Args::toJSON()
 
 static void hashTypeCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
 {
-    for (auto & type : hashTypes)
+    for (auto & type : hashAlgorithms)
         if (hasPrefix(type, prefix))
             completions.add(type);
 }
 
-Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht)
+Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashAlgorithm * ha)
 {
     return Flag {
         .longName = std::move(longName),
         .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')",
         .labels = {"hash-algo"},
-        .handler = {[ht](std::string s) {
-            *ht = parseHashType(s);
+        .handler = {[ha](std::string s) {
+            *ha = parseHashAlgo(s);
         }},
         .completer = hashTypeCompleter,
     };
 }
 
-Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional<HashType> * oht)
+Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha)
 {
     return Flag {
         .longName = std::move(longName),
         .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.",
         .labels = {"hash-algo"},
-        .handler = {[oht](std::string s) {
-            *oht = std::optional<HashType> { parseHashType(s) };
+        .handler = {[oha](std::string s) {
+            *oha = std::optional<HashAlgorithm> {parseHashAlgo(s) };
         }},
         .completer = hashTypeCompleter,
     };
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 72278dccc..0cff76158 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -14,7 +14,7 @@
 
 namespace nix {
 
-enum HashType : char;
+enum struct HashAlgorithm : char;
 
 class MultiCommand;
 
@@ -175,8 +175,8 @@ protected:
 
         std::optional<ExperimentalFeature> experimentalFeature;
 
-        static Flag mkHashTypeFlag(std::string && longName, HashType * ht);
-        static Flag mkHashTypeOptFlag(std::string && longName, std::optional<HashType> * oht);
+        static Flag mkHashTypeFlag(std::string && longName, HashAlgorithm * ha);
+        static Flag mkHashTypeOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha);
     };
 
     /**
diff --git a/src/libutil/git.cc b/src/libutil/git.cc
index a4bd60096..296b75628 100644
--- a/src/libutil/git.cc
+++ b/src/libutil/git.cc
@@ -106,7 +106,7 @@ void parse(
             std::string hashs = getString(source, 20);
             left -= 20;
 
-            Hash hash(htSHA1);
+            Hash hash(HashAlgorithm::SHA1);
             std::copy(hashs.begin(), hashs.end(), hash.hash);
 
             hook(name, TreeEntry {
@@ -241,12 +241,12 @@ Mode dump(
 
 
 TreeEntry dumpHash(
-    HashType ht,
-    SourceAccessor & accessor, const CanonPath & path, PathFilter & filter)
+        HashAlgorithm ha,
+        SourceAccessor & accessor, const CanonPath & path, PathFilter & filter)
 {
     std::function<DumpHook> hook;
     hook = [&](const CanonPath & path) -> TreeEntry {
-        auto hashSink = HashSink(ht);
+        auto hashSink = HashSink(ha);
         auto mode = dump(accessor, path, hashSink, hook, filter);
         auto hash = hashSink.finish().first;
         return {
diff --git a/src/libutil/git.hh b/src/libutil/git.hh
index 303460072..b24b25dd3 100644
--- a/src/libutil/git.hh
+++ b/src/libutil/git.hh
@@ -123,9 +123,9 @@ Mode dump(
  * A smaller wrapper around `dump`.
  */
 TreeEntry dumpHash(
-    HashType ht,
-    SourceAccessor & accessor, const CanonPath & path,
-    PathFilter & filter = defaultPathFilter);
+            HashAlgorithm ha,
+            SourceAccessor & accessor, const CanonPath & path,
+            PathFilter & filter = defaultPathFilter);
 
 /**
  * A line from the output of `git ls-remote --symref`.
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 144f7ae7e..38a29c459 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -16,23 +16,23 @@
 
 namespace nix {
 
-static size_t regularHashSize(HashType type) {
+static size_t regularHashSize(HashAlgorithm type) {
     switch (type) {
-    case htMD5: return md5HashSize;
-    case htSHA1: return sha1HashSize;
-    case htSHA256: return sha256HashSize;
-    case htSHA512: return sha512HashSize;
+    case HashAlgorithm::MD5: return md5HashSize;
+    case HashAlgorithm::SHA1: return sha1HashSize;
+    case HashAlgorithm::SHA256: return sha256HashSize;
+    case HashAlgorithm::SHA512: return sha512HashSize;
     }
     abort();
 }
 
 
-std::set<std::string> hashTypes = { "md5", "sha1", "sha256", "sha512" };
+std::set<std::string> hashAlgorithms = {"md5", "sha1", "sha256", "sha512" };
 
 
-Hash::Hash(HashType type) : type(type)
+Hash::Hash(HashAlgorithm algo) : algo(algo)
 {
-    hashSize = regularHashSize(type);
+    hashSize = regularHashSize(algo);
     assert(hashSize <= maxHashSize);
     memset(hash, 0, maxHashSize);
 }
@@ -109,16 +109,16 @@ static std::string printHash32(const Hash & hash)
 
 std::string printHash16or32(const Hash & hash)
 {
-    assert(hash.type);
-    return hash.to_string(hash.type == htMD5 ? HashFormat::Base16 : HashFormat::Base32, false);
+    assert(static_cast<char>(hash.algo));
+    return hash.to_string(hash.algo == HashAlgorithm::MD5 ? HashFormat::Base16 : HashFormat::Base32, false);
 }
 
 
-std::string Hash::to_string(HashFormat hashFormat, bool includeType) const
+std::string Hash::to_string(HashFormat hashFormat, bool includeAlgo) const
 {
     std::string s;
-    if (hashFormat == HashFormat::SRI || includeType) {
-        s += printHashType(type);
+    if (hashFormat == HashFormat::SRI || includeAlgo) {
+        s += printHashAlgo(algo);
         s += hashFormat == HashFormat::SRI ? '-' : ':';
     }
     switch (hashFormat) {
@@ -136,7 +136,7 @@ std::string Hash::to_string(HashFormat hashFormat, bool includeType) const
     return s;
 }
 
-Hash Hash::dummy(htSHA256);
+Hash Hash::dummy(HashAlgorithm::SHA256);
 
 Hash Hash::parseSRI(std::string_view original) {
     auto rest = original;
@@ -145,18 +145,18 @@ Hash Hash::parseSRI(std::string_view original) {
     auto hashRaw = splitPrefixTo(rest, '-');
     if (!hashRaw)
         throw BadHash("hash '%s' is not SRI", original);
-    HashType parsedType = parseHashType(*hashRaw);
+    HashAlgorithm parsedType = parseHashAlgo(*hashRaw);
 
     return Hash(rest, parsedType, true);
 }
 
 // Mutates the string to eliminate the prefixes when found
-static std::pair<std::optional<HashType>, bool> getParsedTypeAndSRI(std::string_view & rest)
+static std::pair<std::optional<HashAlgorithm>, bool> getParsedTypeAndSRI(std::string_view & rest)
 {
     bool isSRI = false;
 
     // Parse the hash type before the separator, if there was one.
-    std::optional<HashType> optParsedType;
+    std::optional<HashAlgorithm> optParsedType;
     {
         auto hashRaw = splitPrefixTo(rest, ':');
 
@@ -166,7 +166,7 @@ static std::pair<std::optional<HashType>, bool> getParsedTypeAndSRI(std::string_
                 isSRI = true;
         }
         if (hashRaw)
-            optParsedType = parseHashType(*hashRaw);
+            optParsedType = parseHashAlgo(*hashRaw);
     }
 
     return {optParsedType, isSRI};
@@ -185,29 +185,29 @@ Hash Hash::parseAnyPrefixed(std::string_view original)
     return Hash(rest, *optParsedType, isSRI);
 }
 
-Hash Hash::parseAny(std::string_view original, std::optional<HashType> optType)
+Hash Hash::parseAny(std::string_view original, std::optional<HashAlgorithm> optAlgo)
 {
     auto rest = original;
     auto [optParsedType, isSRI] = getParsedTypeAndSRI(rest);
 
     // Either the string or user must provide the type, if they both do they
     // must agree.
-    if (!optParsedType && !optType)
+    if (!optParsedType && !optAlgo)
         throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context", rest);
-    else if (optParsedType && optType && *optParsedType != *optType)
-        throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType));
+    else if (optParsedType && optAlgo && *optParsedType != *optAlgo)
+        throw BadHash("hash '%s' should have type '%s'", original, printHashAlgo(*optAlgo));
 
-    HashType hashType = optParsedType ? *optParsedType : *optType;
-    return Hash(rest, hashType, isSRI);
+    HashAlgorithm hashAlgo = optParsedType ? *optParsedType : *optAlgo;
+    return Hash(rest, hashAlgo, isSRI);
 }
 
-Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashType type)
+Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashAlgorithm algo)
 {
-    return Hash(s, type, false);
+    return Hash(s, algo, false);
 }
 
-Hash::Hash(std::string_view rest, HashType type, bool isSRI)
-    : Hash(type)
+Hash::Hash(std::string_view rest, HashAlgorithm algo, bool isSRI)
+    : Hash(algo)
 {
     if (!isSRI && rest.size() == base16Len()) {
 
@@ -257,19 +257,19 @@ Hash::Hash(std::string_view rest, HashType type, bool isSRI)
     }
 
     else
-        throw BadHash("hash '%s' has wrong length for hash type '%s'", rest, printHashType(this->type));
+        throw BadHash("hash '%s' has wrong length for hash algorithm '%s'", rest, printHashAlgo(this->algo));
 }
 
-Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashType> ht)
+Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashAlgorithm> ha)
 {
     if (hashStr.empty()) {
-        if (!ht)
+        if (!ha)
             throw BadHash("empty hash requires explicit hash type");
-        Hash h(*ht);
+        Hash h(*ha);
         warn("found empty hash, assuming '%s'", h.to_string(HashFormat::SRI, true));
         return h;
     } else
-        return Hash::parseAny(hashStr, ht);
+        return Hash::parseAny(hashStr, ha);
 }
 
 
@@ -282,58 +282,58 @@ union Ctx
 };
 
 
-static void start(HashType ht, Ctx & ctx)
+static void start(HashAlgorithm ha, Ctx & ctx)
 {
-    if (ht == htMD5) MD5_Init(&ctx.md5);
-    else if (ht == htSHA1) SHA1_Init(&ctx.sha1);
-    else if (ht == htSHA256) SHA256_Init(&ctx.sha256);
-    else if (ht == htSHA512) SHA512_Init(&ctx.sha512);
+    if (ha == HashAlgorithm::MD5) MD5_Init(&ctx.md5);
+    else if (ha == HashAlgorithm::SHA1) SHA1_Init(&ctx.sha1);
+    else if (ha == HashAlgorithm::SHA256) SHA256_Init(&ctx.sha256);
+    else if (ha == HashAlgorithm::SHA512) SHA512_Init(&ctx.sha512);
 }
 
 
-static void update(HashType ht, Ctx & ctx,
-    std::string_view data)
+static void update(HashAlgorithm ha, Ctx & ctx,
+                   std::string_view data)
 {
-    if (ht == htMD5) MD5_Update(&ctx.md5, data.data(), data.size());
-    else if (ht == htSHA1) SHA1_Update(&ctx.sha1, data.data(), data.size());
-    else if (ht == htSHA256) SHA256_Update(&ctx.sha256, data.data(), data.size());
-    else if (ht == htSHA512) SHA512_Update(&ctx.sha512, data.data(), data.size());
+    if (ha == HashAlgorithm::MD5) MD5_Update(&ctx.md5, data.data(), data.size());
+    else if (ha == HashAlgorithm::SHA1) SHA1_Update(&ctx.sha1, data.data(), data.size());
+    else if (ha == HashAlgorithm::SHA256) SHA256_Update(&ctx.sha256, data.data(), data.size());
+    else if (ha == HashAlgorithm::SHA512) SHA512_Update(&ctx.sha512, data.data(), data.size());
 }
 
 
-static void finish(HashType ht, Ctx & ctx, unsigned char * hash)
+static void finish(HashAlgorithm ha, Ctx & ctx, unsigned char * hash)
 {
-    if (ht == htMD5) MD5_Final(hash, &ctx.md5);
-    else if (ht == htSHA1) SHA1_Final(hash, &ctx.sha1);
-    else if (ht == htSHA256) SHA256_Final(hash, &ctx.sha256);
-    else if (ht == htSHA512) SHA512_Final(hash, &ctx.sha512);
+    if (ha == HashAlgorithm::MD5) MD5_Final(hash, &ctx.md5);
+    else if (ha == HashAlgorithm::SHA1) SHA1_Final(hash, &ctx.sha1);
+    else if (ha == HashAlgorithm::SHA256) SHA256_Final(hash, &ctx.sha256);
+    else if (ha == HashAlgorithm::SHA512) SHA512_Final(hash, &ctx.sha512);
 }
 
 
-Hash hashString(HashType ht, std::string_view s)
+Hash hashString(HashAlgorithm ha, std::string_view s)
 {
     Ctx ctx;
-    Hash hash(ht);
-    start(ht, ctx);
-    update(ht, ctx, s);
-    finish(ht, ctx, hash.hash);
+    Hash hash(ha);
+    start(ha, ctx);
+    update(ha, ctx, s);
+    finish(ha, ctx, hash.hash);
     return hash;
 }
 
 
-Hash hashFile(HashType ht, const Path & path)
+Hash hashFile(HashAlgorithm ha, const Path & path)
 {
-    HashSink sink(ht);
+    HashSink sink(ha);
     readFile(path, sink);
     return sink.finish().first;
 }
 
 
-HashSink::HashSink(HashType ht) : ht(ht)
+HashSink::HashSink(HashAlgorithm ha) : ha(ha)
 {
     ctx = new Ctx;
     bytes = 0;
-    start(ht, *ctx);
+    start(ha, *ctx);
 }
 
 HashSink::~HashSink()
@@ -345,14 +345,14 @@ HashSink::~HashSink()
 void HashSink::writeUnbuffered(std::string_view data)
 {
     bytes += data.size();
-    update(ht, *ctx, data);
+    update(ha, *ctx, data);
 }
 
 HashResult HashSink::finish()
 {
     flush();
-    Hash hash(ht);
-    nix::finish(ht, *ctx, hash.hash);
+    Hash hash(ha);
+    nix::finish(ha, *ctx, hash.hash);
     return HashResult(hash, bytes);
 }
 
@@ -360,16 +360,16 @@ HashResult HashSink::currentHash()
 {
     flush();
     Ctx ctx2 = *ctx;
-    Hash hash(ht);
-    nix::finish(ht, ctx2, hash.hash);
+    Hash hash(ha);
+    nix::finish(ha, ctx2, hash.hash);
     return HashResult(hash, bytes);
 }
 
 
 HashResult hashPath(
-    HashType ht, const Path & path, PathFilter & filter)
+        HashAlgorithm ha, const Path & path, PathFilter & filter)
 {
-    HashSink sink(ht);
+    HashSink sink(ha);
     dumpPath(path, sink, filter);
     return sink.finish();
 }
@@ -377,7 +377,7 @@ HashResult hashPath(
 
 Hash compressHash(const Hash & hash, unsigned int newSize)
 {
-    Hash h(hash.type);
+    Hash h(hash.algo);
     h.hashSize = newSize;
     for (unsigned int i = 0; i < hash.hashSize; ++i)
         h.hash[i % newSize] ^= hash.hash[i];
@@ -420,31 +420,31 @@ std::string_view printHashFormat(HashFormat HashFormat)
     }
 }
 
-std::optional<HashType> parseHashTypeOpt(std::string_view s)
+std::optional<HashAlgorithm> parseHashAlgoOpt(std::string_view s)
 {
-    if (s == "md5") return htMD5;
-    if (s == "sha1") return htSHA1;
-    if (s == "sha256") return htSHA256;
-    if (s == "sha512") return htSHA512;
+    if (s == "md5") return HashAlgorithm::MD5;
+    if (s == "sha1") return HashAlgorithm::SHA1;
+    if (s == "sha256") return HashAlgorithm::SHA256;
+    if (s == "sha512") return HashAlgorithm::SHA512;
     return std::nullopt;
 }
 
-HashType parseHashType(std::string_view s)
+HashAlgorithm parseHashAlgo(std::string_view s)
 {
-    auto opt_h = parseHashTypeOpt(s);
+    auto opt_h = parseHashAlgoOpt(s);
     if (opt_h)
         return *opt_h;
     else
         throw UsageError("unknown hash algorithm '%1%', expect 'md5', 'sha1', 'sha256', or 'sha512'", s);
 }
 
-std::string_view printHashType(HashType ht)
+std::string_view printHashAlgo(HashAlgorithm ha)
 {
-    switch (ht) {
-    case htMD5: return "md5";
-    case htSHA1: return "sha1";
-    case htSHA256: return "sha256";
-    case htSHA512: return "sha512";
+    switch (ha) {
+    case HashAlgorithm::MD5: return "md5";
+    case HashAlgorithm::SHA1: return "sha1";
+    case HashAlgorithm::SHA256: return "sha256";
+    case HashAlgorithm::SHA512: return "sha512";
     default:
         // illegal hash type enum value internally, as opposed to external input
         // which should be validated with nice error message.
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index 820154e7a..3c97ed4b1 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -12,7 +12,7 @@ namespace nix {
 MakeError(BadHash, Error);
 
 
-enum HashType : char { htMD5 = 42, htSHA1, htSHA256, htSHA512 };
+enum struct HashAlgorithm : char { MD5 = 42, SHA1, SHA256, SHA512 };
 
 
 const int md5HashSize = 16;
@@ -20,7 +20,7 @@ const int sha1HashSize = 20;
 const int sha256HashSize = 32;
 const int sha512HashSize = 64;
 
-extern std::set<std::string> hashTypes;
+extern std::set<std::string> hashAlgorithms;
 
 extern const std::string base32Chars;
 
@@ -46,12 +46,12 @@ struct Hash
     size_t hashSize = 0;
     uint8_t hash[maxHashSize] = {};
 
-    HashType type;
+    HashAlgorithm algo;
 
     /**
      * Create a zero-filled hash object.
      */
-    explicit Hash(HashType type);
+    explicit Hash(HashAlgorithm algo);
 
     /**
      * Parse the hash from a string representation in the format
@@ -60,7 +60,7 @@ struct Hash
      * is not present, then the hash type must be specified in the
      * string.
      */
-    static Hash parseAny(std::string_view s, std::optional<HashType> type);
+    static Hash parseAny(std::string_view s, std::optional<HashAlgorithm> optAlgo);
 
     /**
      * Parse a hash from a string representation like the above, except the
@@ -72,7 +72,7 @@ struct Hash
      * Parse a plain hash that musst not have any prefix indicating the type.
      * The type is passed in to disambiguate.
      */
-    static Hash parseNonSRIUnprefixed(std::string_view s, HashType type);
+    static Hash parseNonSRIUnprefixed(std::string_view s, HashAlgorithm algo);
 
     static Hash parseSRI(std::string_view original);
 
@@ -81,7 +81,7 @@ private:
      * The type must be provided, the string view must not include <type>
      * prefix. `isSRI` helps disambigate the various base-* encodings.
      */
-    Hash(std::string_view s, HashType type, bool isSRI);
+    Hash(std::string_view s, HashAlgorithm algo, bool isSRI);
 
 public:
     /**
@@ -125,10 +125,10 @@ public:
 
     /**
      * Return a string representation of the hash, in base-16, base-32
-     * or base-64. By default, this is prefixed by the hash type
+     * or base-64. By default, this is prefixed by the hash algo
      * (e.g. "sha256:").
      */
-    [[nodiscard]] std::string to_string(HashFormat hashFormat, bool includeType) const;
+    [[nodiscard]] std::string to_string(HashFormat hashFormat, bool includeAlgo) const;
 
     [[nodiscard]] std::string gitRev() const
     {
@@ -146,7 +146,7 @@ public:
 /**
  * Helper that defaults empty hashes to the 0 hash.
  */
-Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashType> ht);
+Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashAlgorithm> ha);
 
 /**
  * Print a hash in base-16 if it's MD5, or base-32 otherwise.
@@ -156,14 +156,14 @@ std::string printHash16or32(const Hash & hash);
 /**
  * Compute the hash of the given string.
  */
-Hash hashString(HashType ht, std::string_view s);
+Hash hashString(HashAlgorithm ha, std::string_view s);
 
 /**
  * Compute the hash of the given file, hashing its contents directly.
  *
  * (Metadata, such as the executable permission bit, is ignored.)
  */
-Hash hashFile(HashType ht, const Path & path);
+Hash hashFile(HashAlgorithm ha, const Path & path);
 
 /**
  * Compute the hash of the given path, serializing as a Nix Archive and
@@ -172,8 +172,8 @@ Hash hashFile(HashType ht, const Path & path);
  * The hash is defined as (essentially) hashString(ht, dumpPath(path)).
  */
 typedef std::pair<Hash, uint64_t> HashResult;
-HashResult hashPath(HashType ht, const Path & path,
-    PathFilter & filter = defaultPathFilter);
+HashResult hashPath(HashAlgorithm ha, const Path & path,
+                    PathFilter & filter = defaultPathFilter);
 
 /**
  * Compress a hash to the specified number of bytes by cyclically
@@ -199,17 +199,17 @@ std::string_view printHashFormat(HashFormat hashFormat);
 /**
  * Parse a string representing a hash type.
  */
-HashType parseHashType(std::string_view s);
+HashAlgorithm parseHashAlgo(std::string_view s);
 
 /**
  * Will return nothing on parse error
  */
-std::optional<HashType> parseHashTypeOpt(std::string_view s);
+std::optional<HashAlgorithm> parseHashAlgoOpt(std::string_view s);
 
 /**
  * And the reverse.
  */
-std::string_view printHashType(HashType ht);
+std::string_view printHashAlgo(HashAlgorithm ha);
 
 
 union Ctx;
@@ -222,12 +222,12 @@ struct AbstractHashSink : virtual Sink
 class HashSink : public BufferedSink, public AbstractHashSink
 {
 private:
-    HashType ht;
+    HashAlgorithm ha;
     Ctx * ctx;
     uint64_t bytes;
 
 public:
-    HashSink(HashType ht);
+    HashSink(HashAlgorithm ha);
     HashSink(const HashSink & h);
     ~HashSink();
     void writeUnbuffered(std::string_view data) override;
diff --git a/src/libutil/references.cc b/src/libutil/references.cc
index 9d75606ef..d82d51945 100644
--- a/src/libutil/references.cc
+++ b/src/libutil/references.cc
@@ -110,8 +110,8 @@ void RewritingSink::flush()
     prev.clear();
 }
 
-HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus)
-    : hashSink(ht)
+HashModuloSink::HashModuloSink(HashAlgorithm ha, const std::string & modulus)
+    : hashSink(ha)
     , rewritingSink(modulus, std::string(modulus.size(), 0), hashSink)
 {
 }
diff --git a/src/libutil/references.hh b/src/libutil/references.hh
index f0baeffe1..8bc9f7ec9 100644
--- a/src/libutil/references.hh
+++ b/src/libutil/references.hh
@@ -46,7 +46,7 @@ struct HashModuloSink : AbstractHashSink
     HashSink hashSink;
     RewritingSink rewritingSink;
 
-    HashModuloSink(HashType ht, const std::string & modulus);
+    HashModuloSink(HashAlgorithm ha, const std::string & modulus);
 
     void operator () (std::string_view data) override;
 
diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc
index 7813433a7..afbbbe1a9 100644
--- a/src/libutil/source-accessor.cc
+++ b/src/libutil/source-accessor.cc
@@ -39,11 +39,11 @@ void SourceAccessor::readFile(
 }
 
 Hash SourceAccessor::hashPath(
-    const CanonPath & path,
-    PathFilter & filter,
-    HashType ht)
+        const CanonPath & path,
+        PathFilter & filter,
+        HashAlgorithm ha)
 {
-    HashSink sink(ht);
+    HashSink sink(ha);
     dumpPath(path, sink, filter);
     return sink.finish().first;
 }
diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh
index 264caab16..3ca12d624 100644
--- a/src/libutil/source-accessor.hh
+++ b/src/libutil/source-accessor.hh
@@ -97,9 +97,9 @@ struct SourceAccessor
         PathFilter & filter = defaultPathFilter);
 
     Hash hashPath(
-        const CanonPath & path,
-        PathFilter & filter = defaultPathFilter,
-        HashType ht = htSHA256);
+            const CanonPath & path,
+            PathFilter & filter = defaultPathFilter,
+            HashAlgorithm ha = HashAlgorithm::SHA256);
 
     /**
      * Return a corresponding path in the root filesystem, if
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 25f0107bc..75ad4e75f 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -193,7 +193,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
     if (opArgs.empty())
         throw UsageError("first argument must be hash algorithm");
 
-    HashType hashAlgo = parseHashType(opArgs.front());
+    HashAlgorithm hashAlgo = parseHashAlgo(opArgs.front());
     opArgs.pop_front();
 
     for (auto & i : opArgs)
@@ -214,7 +214,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs)
         throw UsageError("'--print-fixed-path' requires three arguments");
 
     Strings::iterator i = opArgs.begin();
-    HashType hashAlgo = parseHashType(*i++);
+    HashAlgorithm hashAlgo = parseHashAlgo(*i++);
     std::string hash = *i++;
     std::string name = *i++;
 
@@ -405,7 +405,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
                 for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) {
                     auto info = store->queryPathInfo(j);
                     if (query == qHash) {
-                        assert(info->narHash.type == htSHA256);
+                        assert(info->narHash.algo == HashAlgorithm::SHA256);
                         cout << fmt("%s\n", info->narHash.to_string(HashFormat::Base32, true));
                     } else if (query == qSize)
                         cout << fmt("%d\n", info->narSize);
@@ -541,7 +541,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise)
             if (canonicalise)
                 canonicalisePathMetaData(store->printStorePath(info->path), {});
             if (!hashGiven) {
-                HashResult hash = hashPath(htSHA256, store->printStorePath(info->path));
+                HashResult hash = hashPath(HashAlgorithm::SHA256, store->printStorePath(info->path));
                 info->narHash = hash.first;
                 info->narSize = hash.second;
             }
@@ -763,7 +763,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
         auto path = store->followLinksToStorePath(i);
         printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path));
         auto info = store->queryPathInfo(path);
-        HashSink sink(info->narHash.type);
+        HashSink sink(info->narHash.algo);
         store->narFromPath(path, sink);
         auto current = sink.finish();
         if (current.first != info->narHash) {
@@ -979,7 +979,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                 auto deriver = readString(in);
                 ValidPathInfo info {
                     store->parseStorePath(path),
-                    Hash::parseAny(readString(in), htSHA256),
+                    Hash::parseAny(readString(in), HashAlgorithm::SHA256),
                 };
                 if (deriver != "")
                     info.deriver = store->parseStorePath(deriver);
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index f9d487ada..02de796b5 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -60,11 +60,11 @@ struct CmdAddToStore : MixDryRun, StoreCommand
         StringSink sink;
         dumpPath(path, sink);
 
-        auto narHash = hashString(htSHA256, sink.s);
+        auto narHash = hashString(HashAlgorithm::SHA256, sink.s);
 
         Hash hash = narHash;
         if (ingestionMethod == FileIngestionMethod::Flat) {
-            HashSink hsink(htSHA256);
+            HashSink hsink(HashAlgorithm::SHA256);
             readFile(path, hsink);
             hash = hsink.finish().first;
         }
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 62f96ef1d..638178afa 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -18,7 +18,7 @@ struct CmdHashBase : Command
     FileIngestionMethod mode;
     HashFormat hashFormat = HashFormat::SRI;
     bool truncate = false;
-    HashType ht = htSHA256;
+    HashAlgorithm ha = HashAlgorithm::SHA256;
     std::vector<std::string> paths;
     std::optional<std::string> modulus;
 
@@ -48,7 +48,7 @@ struct CmdHashBase : Command
             .handler = {&hashFormat, HashFormat::Base16},
         });
 
-        addFlag(Flag::mkHashTypeFlag("type", &ht));
+        addFlag(Flag::mkHashTypeFlag("type", &ha));
 
         #if 0
         addFlag({
@@ -84,9 +84,9 @@ struct CmdHashBase : Command
 
             std::unique_ptr<AbstractHashSink> hashSink;
             if (modulus)
-                hashSink = std::make_unique<HashModuloSink>(ht, *modulus);
+                hashSink = std::make_unique<HashModuloSink>(ha, *modulus);
             else
-                hashSink = std::make_unique<HashSink>(ht);
+                hashSink = std::make_unique<HashSink>(ha);
 
             switch (mode) {
             case FileIngestionMethod::Flat:
@@ -107,7 +107,7 @@ struct CmdHashBase : Command
 struct CmdToBase : Command
 {
     HashFormat hashFormat;
-    std::optional<HashType> ht;
+    std::optional<HashAlgorithm> ht;
     std::vector<std::string> args;
 
     CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat)
@@ -139,7 +139,7 @@ struct CmdHashConvert : Command
 {
     std::optional<HashFormat> from;
     HashFormat to;
-    std::optional<HashType> type;
+    std::optional<HashAlgorithm> type;
     std::vector<std::string> hashStrings;
 
     CmdHashConvert(): to(HashFormat::SRI) {
@@ -166,7 +166,7 @@ struct CmdHashConvert : Command
             .description = "Specify the algorithm if it can't be auto-detected.",
             .labels = {"hash algorithm"},
             .handler = {[this](std::string str) {
-                type = parseHashType(str);
+                type = parseHashAlgo(str);
             }},
         });
         expectArgs({
@@ -223,7 +223,7 @@ static auto rCmdHash = registerCommand<CmdHash>("hash");
 /* Legacy nix-hash command. */
 static int compatNixHash(int argc, char * * argv)
 {
-    std::optional<HashType> ht;
+    std::optional<HashAlgorithm> ha;
     bool flat = false;
     HashFormat hashFormat = HashFormat::Base16;
     bool truncate = false;
@@ -243,7 +243,7 @@ static int compatNixHash(int argc, char * * argv)
         else if (*arg == "--truncate") truncate = true;
         else if (*arg == "--type") {
             std::string s = getArg(*arg, arg, end);
-            ht = parseHashType(s);
+            ha = parseHashAlgo(s);
         }
         else if (*arg == "--to-base16") {
             op = opTo;
@@ -270,8 +270,8 @@ static int compatNixHash(int argc, char * * argv)
 
     if (op == opHash) {
         CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive);
-        if (!ht.has_value()) ht = htMD5;
-        cmd.ht = ht.value();
+        if (!ha.has_value()) ha = HashAlgorithm::MD5;
+        cmd.ha = ha.value();
         cmd.hashFormat = hashFormat;
         cmd.truncate = truncate;
         cmd.paths = ss;
@@ -281,7 +281,7 @@ static int compatNixHash(int argc, char * * argv)
     else {
         CmdToBase cmd(hashFormat);
         cmd.args = ss;
-        if (ht.has_value()) cmd.ht = ht;
+        if (ha.has_value()) cmd.ht = ha;
         cmd.run();
     }
 
diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc
index 3ed7946a8..09f33a51e 100644
--- a/src/nix/prefetch.cc
+++ b/src/nix/prefetch.cc
@@ -46,13 +46,13 @@ std::string resolveMirrorUrl(EvalState & state, const std::string & url)
 }
 
 std::tuple<StorePath, Hash> prefetchFile(
-    ref<Store> store,
-    std::string_view url,
-    std::optional<std::string> name,
-    HashType hashType,
-    std::optional<Hash> expectedHash,
-    bool unpack,
-    bool executable)
+        ref<Store> store,
+        std::string_view url,
+        std::optional<std::string> name,
+        HashAlgorithm hashAlgo,
+        std::optional<Hash> expectedHash,
+        bool unpack,
+        bool executable)
 {
     auto ingestionMethod = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
 
@@ -69,7 +69,7 @@ std::tuple<StorePath, Hash> prefetchFile(
     /* If an expected hash is given, the file may already exist in
        the store. */
     if (expectedHash) {
-        hashType = expectedHash->type;
+        hashAlgo = expectedHash->algo;
         storePath = store->makeFixedOutputPath(*name, FixedOutputInfo {
             .method = ingestionMethod,
             .hash = *expectedHash,
@@ -122,7 +122,7 @@ std::tuple<StorePath, Hash> prefetchFile(
         Activity act(*logger, lvlChatty, actUnknown,
             fmt("adding '%s' to the store", url));
 
-        auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash);
+        auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashAlgo, expectedHash);
         storePath = info.path;
         assert(info.ca);
         hash = info.ca->hash;
@@ -134,7 +134,7 @@ std::tuple<StorePath, Hash> prefetchFile(
 static int main_nix_prefetch_url(int argc, char * * argv)
 {
     {
-        HashType ht = htSHA256;
+        HashAlgorithm ha = HashAlgorithm::SHA256;
         std::vector<std::string> args;
         bool printPath = getEnv("PRINT_PATH") == "1";
         bool fromExpr = false;
@@ -155,7 +155,7 @@ static int main_nix_prefetch_url(int argc, char * * argv)
                 printVersion("nix-prefetch-url");
             else if (*arg == "--type") {
                 auto s = getArg(*arg, arg, end);
-                ht = parseHashType(s);
+                ha = parseHashAlgo(s);
             }
             else if (*arg == "--print-path")
                 printPath = true;
@@ -233,10 +233,10 @@ static int main_nix_prefetch_url(int argc, char * * argv)
 
         std::optional<Hash> expectedHash;
         if (args.size() == 2)
-            expectedHash = Hash::parseAny(args[1], ht);
+            expectedHash = Hash::parseAny(args[1], ha);
 
         auto [storePath, hash] = prefetchFile(
-            store, resolveMirrorUrl(*state, url), name, ht, expectedHash, unpack, executable);
+            store, resolveMirrorUrl(*state, url), name, ha, expectedHash, unpack, executable);
 
         stopProgressBar();
 
@@ -258,7 +258,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON
     std::string url;
     bool executable = false;
     std::optional<std::string> name;
-    HashType hashType = htSHA256;
+    HashAlgorithm hashAlgo = HashAlgorithm::SHA256;
     std::optional<Hash> expectedHash;
 
     CmdStorePrefetchFile()
@@ -275,11 +275,11 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON
             .description = "The expected hash of the file.",
             .labels = {"hash"},
             .handler = {[&](std::string s) {
-                expectedHash = Hash::parseAny(s, hashType);
+                expectedHash = Hash::parseAny(s, hashAlgo);
             }}
         });
 
-        addFlag(Flag::mkHashTypeFlag("hash-type", &hashType));
+        addFlag(Flag::mkHashTypeFlag("hash-type", &hashAlgo));
 
         addFlag({
             .longName = "executable",
@@ -305,7 +305,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON
     }
     void run(ref<Store> store) override
     {
-        auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, false, executable);
+        auto [storePath, hash] = prefetchFile(store, url, name, hashAlgo, expectedHash, false, executable);
 
         if (json) {
             auto res = nlohmann::json::object();
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 147b4680b..9d9492da9 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -216,7 +216,7 @@ struct ProfileManifest
         StringSink sink;
         dumpPath(tempDir, sink);
 
-        auto narHash = hashString(htSHA256, sink.s);
+        auto narHash = hashString(HashAlgorithm::SHA256, sink.s);
 
         ValidPathInfo info {
             *store,
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 78cb765ce..cd0f6d95f 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -98,7 +98,7 @@ struct CmdVerify : StorePathsCommand
 
                 if (!noContents) {
 
-                    auto hashSink = HashSink(info->narHash.type);
+                    auto hashSink = HashSink(info->narHash.algo);
 
                     store->narFromPath(info->path, hashSink);
 
diff --git a/tests/unit/libstore/common-protocol.cc b/tests/unit/libstore/common-protocol.cc
index c09ac6a3e..d23805fc3 100644
--- a/tests/unit/libstore/common-protocol.cc
+++ b/tests/unit/libstore/common-protocol.cc
@@ -84,15 +84,15 @@ CHARACTERIZATION_TEST(
     (std::tuple<ContentAddress, ContentAddress, ContentAddress> {
         ContentAddress {
             .method = TextIngestionMethod {},
-            .hash = hashString(HashType::htSHA256, "Derive(...)"),
+            .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"),
         },
         ContentAddress {
             .method = FileIngestionMethod::Flat,
-            .hash = hashString(HashType::htSHA1, "blob blob..."),
+            .hash = hashString(HashAlgorithm::SHA1, "blob blob..."),
         },
         ContentAddress {
             .method = FileIngestionMethod::Recursive,
-            .hash = hashString(HashType::htSHA256, "(...)"),
+            .hash = hashString(HashAlgorithm::SHA256, "(...)"),
         },
     }))
 
@@ -179,7 +179,7 @@ CHARACTERIZATION_TEST(
         std::optional {
             ContentAddress {
                 .method = FileIngestionMethod::Flat,
-                .hash = hashString(HashType::htSHA1, "blob blob..."),
+                .hash = hashString(HashAlgorithm::SHA1, "blob blob..."),
             },
         },
     }))
diff --git a/tests/unit/libstore/derivation.cc b/tests/unit/libstore/derivation.cc
index a7f4488fa..7a4b1403a 100644
--- a/tests/unit/libstore/derivation.cc
+++ b/tests/unit/libstore/derivation.cc
@@ -134,7 +134,7 @@ TEST_JSON(DynDerivationTest, caFixedText,
 TEST_JSON(CaDerivationTest, caFloating,
     (DerivationOutput::CAFloating {
         .method = FileIngestionMethod::Recursive,
-        .hashType = htSHA256,
+        .hashAlgo = HashAlgorithm::SHA256,
     }),
     "drv-name", "output-name")
 
@@ -145,7 +145,7 @@ TEST_JSON(DerivationTest, deferred,
 TEST_JSON(ImpureDerivationTest, impure,
     (DerivationOutput::Impure {
         .method = FileIngestionMethod::Recursive,
-        .hashType = htSHA256,
+        .hashAlgo = HashAlgorithm::SHA256,
     }),
     "drv-name", "output-name")
 
diff --git a/tests/unit/libstore/nar-info.cc b/tests/unit/libstore/nar-info.cc
index 4f124e89e..bd10602e7 100644
--- a/tests/unit/libstore/nar-info.cc
+++ b/tests/unit/libstore/nar-info.cc
@@ -26,7 +26,7 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) {
         "foo",
         FixedOutputInfo {
             .method = FileIngestionMethod::Recursive,
-            .hash = hashString(HashType::htSHA256, "(...)"),
+            .hash = hashString(HashAlgorithm::SHA256, "(...)"),
 
             .references = {
                 .others = {
diff --git a/tests/unit/libstore/path-info.cc b/tests/unit/libstore/path-info.cc
index 18f00ca19..80d6fcfed 100644
--- a/tests/unit/libstore/path-info.cc
+++ b/tests/unit/libstore/path-info.cc
@@ -25,7 +25,7 @@ static UnkeyedValidPathInfo makePathInfo(const Store & store, bool includeImpure
         "foo",
         FixedOutputInfo {
             .method = FileIngestionMethod::Recursive,
-            .hash = hashString(HashType::htSHA256, "(...)"),
+            .hash = hashString(HashAlgorithm::SHA256, "(...)"),
 
             .references = {
                 .others = {
diff --git a/tests/unit/libstore/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc
index c8ac87a04..6d2054f7d 100644
--- a/tests/unit/libstore/serve-protocol.cc
+++ b/tests/unit/libstore/serve-protocol.cc
@@ -53,15 +53,15 @@ VERSIONED_CHARACTERIZATION_TEST(
     (std::tuple<ContentAddress, ContentAddress, ContentAddress> {
         ContentAddress {
             .method = TextIngestionMethod {},
-            .hash = hashString(HashType::htSHA256, "Derive(...)"),
+            .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"),
         },
         ContentAddress {
             .method = FileIngestionMethod::Flat,
-            .hash = hashString(HashType::htSHA1, "blob blob..."),
+            .hash = hashString(HashAlgorithm::SHA1, "blob blob..."),
         },
         ContentAddress {
             .method = FileIngestionMethod::Recursive,
-            .hash = hashString(HashType::htSHA256, "(...)"),
+            .hash = hashString(HashAlgorithm::SHA256, "(...)"),
         },
     }))
 
@@ -271,7 +271,7 @@ VERSIONED_CHARACTERIZATION_TEST(
         std::optional {
             ContentAddress {
                 .method = FileIngestionMethod::Flat,
-                .hash = hashString(HashType::htSHA1, "blob blob..."),
+                .hash = hashString(HashAlgorithm::SHA1, "blob blob..."),
             },
         },
     }))
diff --git a/tests/unit/libstore/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc
index ad5943c69..91f804f0c 100644
--- a/tests/unit/libstore/worker-protocol.cc
+++ b/tests/unit/libstore/worker-protocol.cc
@@ -55,15 +55,15 @@ VERSIONED_CHARACTERIZATION_TEST(
     (std::tuple<ContentAddress, ContentAddress, ContentAddress> {
         ContentAddress {
             .method = TextIngestionMethod {},
-            .hash = hashString(HashType::htSHA256, "Derive(...)"),
+            .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"),
         },
         ContentAddress {
             .method = FileIngestionMethod::Flat,
-            .hash = hashString(HashType::htSHA1, "blob blob..."),
+            .hash = hashString(HashAlgorithm::SHA1, "blob blob..."),
         },
         ContentAddress {
             .method = FileIngestionMethod::Recursive,
-            .hash = hashString(HashType::htSHA256, "(...)"),
+            .hash = hashString(HashAlgorithm::SHA256, "(...)"),
         },
     }))
 
@@ -464,7 +464,7 @@ VERSIONED_CHARACTERIZATION_TEST(
                 "foo",
                 FixedOutputInfo {
                     .method = FileIngestionMethod::Recursive,
-                    .hash = hashString(HashType::htSHA256, "(...)"),
+                    .hash = hashString(HashAlgorithm::SHA256, "(...)"),
                     .references = {
                         .others = {
                             StorePath {
@@ -539,7 +539,7 @@ VERSIONED_CHARACTERIZATION_TEST(
         std::optional {
             ContentAddress {
                 .method = FileIngestionMethod::Flat,
-                .hash = hashString(HashType::htSHA1, "blob blob..."),
+                .hash = hashString(HashAlgorithm::SHA1, "blob blob..."),
             },
         },
     }))
diff --git a/tests/unit/libutil/git.cc b/tests/unit/libutil/git.cc
index 551a2d105..141a55816 100644
--- a/tests/unit/libutil/git.cc
+++ b/tests/unit/libutil/git.cc
@@ -95,7 +95,7 @@ const static Tree tree = {
         {
             .mode = Mode::Regular,
             // hello world with special chars from above
-            .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", htSHA1),
+            .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", HashAlgorithm::SHA1),
         },
     },
     {
@@ -103,7 +103,7 @@ const static Tree tree = {
         {
             .mode = Mode::Executable,
             // ditto
-            .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", htSHA1),
+            .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", HashAlgorithm::SHA1),
         },
     },
     {
@@ -111,7 +111,7 @@ const static Tree tree = {
         {
             .mode = Mode::Directory,
             // Empty directory hash
-            .hash = Hash::parseAny("4b825dc642cb6eb9a060e54bf8d69288fbee4904", htSHA1),
+            .hash = Hash::parseAny("4b825dc642cb6eb9a060e54bf8d69288fbee4904", HashAlgorithm::SHA1),
         },
     },
 };
@@ -174,7 +174,7 @@ TEST_F(GitTest, both_roundrip) {
     std::function<DumpHook> dumpHook;
     dumpHook = [&](const CanonPath & path) {
         StringSink s;
-        HashSink hashSink { htSHA1 };
+        HashSink hashSink { HashAlgorithm::SHA1 };
         TeeSink s2 { s, hashSink };
         auto mode = dump(
             files, path, s2, dumpHook,
diff --git a/tests/unit/libutil/hash.cc b/tests/unit/libutil/hash.cc
index 92291afce..4d82c7f09 100644
--- a/tests/unit/libutil/hash.cc
+++ b/tests/unit/libutil/hash.cc
@@ -13,28 +13,28 @@ namespace nix {
     TEST(hashString, testKnownMD5Hashes1) {
         // values taken from: https://tools.ietf.org/html/rfc1321
         auto s1 = "";
-        auto hash = hashString(HashType::htMD5, s1);
+        auto hash = hashString(HashAlgorithm::MD5, s1);
         ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e");
     }
 
     TEST(hashString, testKnownMD5Hashes2) {
         // values taken from: https://tools.ietf.org/html/rfc1321
         auto s2 = "abc";
-        auto hash = hashString(HashType::htMD5, s2);
+        auto hash = hashString(HashAlgorithm::MD5, s2);
         ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72");
     }
 
     TEST(hashString, testKnownSHA1Hashes1) {
         // values taken from: https://tools.ietf.org/html/rfc3174
         auto s = "abc";
-        auto hash = hashString(HashType::htSHA1, s);
+        auto hash = hashString(HashAlgorithm::SHA1, s);
         ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d");
     }
 
     TEST(hashString, testKnownSHA1Hashes2) {
         // values taken from: https://tools.ietf.org/html/rfc3174
         auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq";
-        auto hash = hashString(HashType::htSHA1, s);
+        auto hash = hashString(HashAlgorithm::SHA1, s);
         ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1");
     }
 
@@ -42,7 +42,7 @@ namespace nix {
         // values taken from: https://tools.ietf.org/html/rfc4634
         auto s = "abc";
 
-        auto hash = hashString(HashType::htSHA256, s);
+        auto hash = hashString(HashAlgorithm::SHA256, s);
         ASSERT_EQ(hash.to_string(HashFormat::Base16, true),
                 "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
     }
@@ -50,7 +50,7 @@ namespace nix {
     TEST(hashString, testKnownSHA256Hashes2) {
         // values taken from: https://tools.ietf.org/html/rfc4634
         auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq";
-        auto hash = hashString(HashType::htSHA256, s);
+        auto hash = hashString(HashAlgorithm::SHA256, s);
         ASSERT_EQ(hash.to_string(HashFormat::Base16, true),
                 "sha256:248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1");
     }
@@ -58,7 +58,7 @@ namespace nix {
     TEST(hashString, testKnownSHA512Hashes1) {
         // values taken from: https://tools.ietf.org/html/rfc4634
         auto s = "abc";
-        auto hash = hashString(HashType::htSHA512, s);
+        auto hash = hashString(HashAlgorithm::SHA512, s);
         ASSERT_EQ(hash.to_string(HashFormat::Base16, true),
                 "sha512:ddaf35a193617abacc417349ae20413112e6fa4e89a9"
                 "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd"
@@ -68,7 +68,7 @@ namespace nix {
         // values taken from: https://tools.ietf.org/html/rfc4634
         auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu";
 
-        auto hash = hashString(HashType::htSHA512, s);
+        auto hash = hashString(HashAlgorithm::SHA512, s);
         ASSERT_EQ(hash.to_string(HashFormat::Base16, true),
                 "sha512:8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa1"
                 "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a"

From 837b889c41543b32154ceade2363ec6ad6dff15d Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Tue, 28 Nov 2023 14:41:44 +0100
Subject: [PATCH 176/421] Further HashType renaming + using mkHashAlgoOptFlag
 for new conversion

https://github.com/NixOS/nix/issues/8876
---
 src/libutil/args.cc | 38 +++++++++++++++++++-------------------
 src/libutil/args.hh |  4 ++--
 src/nix/hash.cc     | 17 +++++------------
 src/nix/prefetch.cc |  2 +-
 4 files changed, 27 insertions(+), 34 deletions(-)

diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index ac3727d11..7ea1647d9 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -544,36 +544,36 @@ nlohmann::json Args::toJSON()
     return res;
 }
 
-static void hashTypeCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
+static void hashAlgoCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
 {
     for (auto & type : hashAlgorithms)
         if (hasPrefix(type, prefix))
             completions.add(type);
 }
 
-Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashAlgorithm * ha)
+Args::Flag Args::Flag::mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha)
 {
-    return Flag {
-        .longName = std::move(longName),
-        .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')",
-        .labels = {"hash-algo"},
-        .handler = {[ha](std::string s) {
-            *ha = parseHashAlgo(s);
-        }},
-        .completer = hashTypeCompleter,
+    return Flag{
+            .longName = std::move(longName),
+            .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')",
+            .labels = {"hash-algo"},
+            .handler = {[ha](std::string s) {
+                *ha = parseHashAlgo(s);
+            }},
+            .completer = hashAlgoCompleter,
     };
 }
 
-Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha)
+Args::Flag Args::Flag::mkHashAlgoOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha)
 {
-    return Flag {
-        .longName = std::move(longName),
-        .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.",
-        .labels = {"hash-algo"},
-        .handler = {[oha](std::string s) {
-            *oha = std::optional<HashAlgorithm> {parseHashAlgo(s) };
-        }},
-        .completer = hashTypeCompleter,
+    return Flag{
+            .longName = std::move(longName),
+            .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.",
+            .labels = {"hash-algo"},
+            .handler = {[oha](std::string s) {
+                *oha = std::optional<HashAlgorithm>{parseHashAlgo(s)};
+            }},
+            .completer = hashAlgoCompleter,
     };
 }
 
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 0cff76158..653a9bbd6 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -175,8 +175,8 @@ protected:
 
         std::optional<ExperimentalFeature> experimentalFeature;
 
-        static Flag mkHashTypeFlag(std::string && longName, HashAlgorithm * ha);
-        static Flag mkHashTypeOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha);
+        static Flag mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha);
+        static Flag mkHashAlgoOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha);
     };
 
     /**
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 638178afa..173043c8a 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -48,7 +48,7 @@ struct CmdHashBase : Command
             .handler = {&hashFormat, HashFormat::Base16},
         });
 
-        addFlag(Flag::mkHashTypeFlag("type", &ha));
+        addFlag(Flag::mkHashAlgoFlag("type", &ha));
 
         #if 0
         addFlag({
@@ -112,7 +112,7 @@ struct CmdToBase : Command
 
     CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat)
     {
-        addFlag(Flag::mkHashTypeOptFlag("type", &ht));
+        addFlag(Flag::mkHashAlgoOptFlag("type", &ht));
         expectArgs("strings", &args);
     }
 
@@ -139,7 +139,7 @@ struct CmdHashConvert : Command
 {
     std::optional<HashFormat> from;
     HashFormat to;
-    std::optional<HashAlgorithm> type;
+    std::optional<HashAlgorithm> algo;
     std::vector<std::string> hashStrings;
 
     CmdHashConvert(): to(HashFormat::SRI) {
@@ -161,14 +161,7 @@ struct CmdHashConvert : Command
                 to = parseHashFormat(str);
             }},
         });
-        addFlag({
-            .longName = "algo",
-            .description = "Specify the algorithm if it can't be auto-detected.",
-            .labels = {"hash algorithm"},
-            .handler = {[this](std::string str) {
-                type = parseHashAlgo(str);
-            }},
-        });
+        addFlag(Args::Flag::mkHashAlgoOptFlag("algo", &algo));
         expectArgs({
            .label = "hashes",
            .handler = {&hashStrings},
@@ -184,7 +177,7 @@ struct CmdHashConvert : Command
 
     void run() override {
         for (const auto& s: hashStrings) {
-            Hash h = Hash::parseAny(s, type);
+            Hash h = Hash::parseAny(s, algo);
             if (from && h.to_string(*from, from == HashFormat::SRI) != s) {
                 auto from_as_string = printHashFormat(*from);
                 throw BadHash("input hash '%s' does not have the expected format '--from %s'", s, from_as_string);
diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc
index 09f33a51e..bbfeb8aa4 100644
--- a/src/nix/prefetch.cc
+++ b/src/nix/prefetch.cc
@@ -279,7 +279,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON
             }}
         });
 
-        addFlag(Flag::mkHashTypeFlag("hash-type", &hashAlgo));
+        addFlag(Flag::mkHashAlgoFlag("hash-type", &hashAlgo));
 
         addFlag({
             .longName = "executable",

From fc6f29053aa69b6b14bcad93cb273b1c266e74fe Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Tue, 28 Nov 2023 15:38:15 +0100
Subject: [PATCH 177/421] Renamed HashFormat::Base32 to HashFormat::Nix32

...and also adjusted parsing accordingly.

Also added CLI completion for HashFormats.

https://github.com/NixOS/nix/issues/8876
---
 src/libexpr/primops/fetchTree.cc              |   2 +-
 src/libfetchers/git.cc                        |   2 +-
 src/libfetchers/mercurial.cc                  |   2 +-
 src/libstore/binary-cache-store.cc            |   4 +-
 src/libstore/build/local-derivation-goal.cc   |   2 +-
 src/libstore/content-address.cc               |   2 +-
 src/libstore/derivations.cc                   |   2 +-
 src/libstore/downstream-placeholder.cc        |   4 +-
 src/libstore/export-import.cc                 |   2 +-
 src/libstore/gc.cc                            |   2 +-
 src/libstore/local-store.cc                   |  10 +-
 src/libstore/nar-info-disk-cache.cc           |   4 +-
 src/libstore/nar-info.cc                      |   4 +-
 src/libstore/optimise-store.cc                |   4 +-
 src/libstore/parsed-derivations.cc            |   2 +-
 src/libstore/path-info.cc                     |   6 +-
 src/libstore/path.cc                          |   2 +-
 src/libutil/args.cc                           |  40 ++++++-
 src/libutil/args.hh                           |   3 +
 src/libutil/hash.cc                           |  25 ++--
 src/libutil/hash.hh                           |  10 +-
 src/libutil/references.cc                     |   4 +-
 src/nix-store/nix-store.cc                    |   8 +-
 src/nix/hash.cc                               |  40 +++----
 src/nix/verify.cc                             |   4 +-
 tests/functional/hash.sh                      |   6 +-
 .../lang/eval-okay-convertHash.err.exp        | 108 ++++++++++++++++++
 .../functional/lang/eval-okay-convertHash.exp |   2 +-
 .../functional/lang/eval-okay-convertHash.nix |   2 +
 tests/unit/libutil/hash.cc                    |   2 +-
 30 files changed, 228 insertions(+), 82 deletions(-)
 create mode 100644 tests/functional/lang/eval-okay-convertHash.err.exp

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index ef80c634f..15f870a95 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -304,7 +304,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
             : hashFile(HashAlgorithm::SHA256, state.store->toRealPath(storePath));
         if (hash != *expectedHash)
             state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n  specified: %s\n  got:       %s",
-                *url, expectedHash->to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true)));
+                                                *url, expectedHash->to_string(HashFormat::Nix32, true), hash.to_string(HashFormat::Nix32, true)));
     }
 
     state.allowAndSetStorePathString(storePath, v);
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index a89acc1c0..9e6ba8963 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -52,7 +52,7 @@ bool touchCacheFile(const Path & path, time_t touch_time)
 Path getCachePath(std::string_view key)
 {
     return getCacheDir() + "/nix/gitv3/" +
-        hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Base32, false);
+        hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false);
 }
 
 // Returns the name of the HEAD branch.
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index 713f24bbb..6056b9a3c 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -267,7 +267,7 @@ struct MercurialInputScheme : InputScheme
             }
         }
 
-        Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(HashAlgorithm::SHA256, actualUrl).to_string(HashFormat::Base32, false));
+        Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(HashAlgorithm::SHA256, actualUrl).to_string(HashFormat::Nix32, false));
 
         /* If this is a commit hash that we already have, we don't
            have to pull again. */
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index f287d72a8..2837e8934 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -165,8 +165,8 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     auto [fileHash, fileSize] = fileHashSink.finish();
     narInfo->fileHash = fileHash;
     narInfo->fileSize = fileSize;
-    narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Base32, false) + ".nar"
-        + (compression == "xz" ? ".xz" :
+    narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Nix32, false) + ".nar"
+                   + (compression == "xz" ? ".xz" :
            compression == "bzip2" ? ".bz2" :
            compression == "zstd" ? ".zst" :
            compression == "lzip" ? ".lzip" :
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 4c3dc1f5c..802b39f84 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -1067,7 +1067,7 @@ void LocalDerivationGoal::initTmpDir() {
                 env[i.first] = i.second;
             } else {
                 auto hash = hashString(HashAlgorithm::SHA256, i.first);
-                std::string fn = ".attr-" + hash.to_string(HashFormat::Base32, false);
+                std::string fn = ".attr-" + hash.to_string(HashFormat::Nix32, false);
                 Path p = tmpDir + "/" + fn;
                 writeFile(p, rewriteStrings(i.second, inputRewrites));
                 chownToBuilder(p);
diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc
index de8194f73..f42a13126 100644
--- a/src/libstore/content-address.cc
+++ b/src/libstore/content-address.cc
@@ -61,7 +61,7 @@ std::string ContentAddress::render() const
                 + makeFileIngestionPrefix(method);
         },
     }, method.raw)
-        + this->hash.to_string(HashFormat::Base32, true);
+        + this->hash.to_string(HashFormat::Nix32, true);
 }
 
 /**
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index c68631c1a..664ab7556 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -958,7 +958,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva
 std::string hashPlaceholder(const OutputNameView outputName)
 {
     // FIXME: memoize?
-    return "/" + hashString(HashAlgorithm::SHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Base32, false);
+    return "/" + hashString(HashAlgorithm::SHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Nix32, false);
 }
 
 
diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc
index 10df37fa4..91d47f946 100644
--- a/src/libstore/downstream-placeholder.cc
+++ b/src/libstore/downstream-placeholder.cc
@@ -5,7 +5,7 @@ namespace nix {
 
 std::string DownstreamPlaceholder::render() const
 {
-    return "/" + hash.to_string(HashFormat::Base32, false);
+    return "/" + hash.to_string(HashFormat::Nix32, false);
 }
 
 
@@ -31,7 +31,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation(
     xpSettings.require(Xp::DynamicDerivations);
     auto compressed = compressHash(placeholder.hash, 20);
     auto clearText = "nix-computed-output:"
-        + compressed.to_string(HashFormat::Base32, false)
+        + compressed.to_string(HashFormat::Nix32, false)
         + ":" + std::string { outputName };
     return DownstreamPlaceholder {
         hashString(HashAlgorithm::SHA256, clearText)
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index 48718ef84..d57b25bd7 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -41,7 +41,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
     Hash hash = hashSink.currentHash().first;
     if (hash != info->narHash && info->narHash != Hash(info->narHash.algo))
         throw Error("hash of path '%s' has changed from '%s' to '%s'!",
-            printStorePath(path), info->narHash.to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true));
+                    printStorePath(path), info->narHash.to_string(HashFormat::Nix32, true), hash.to_string(HashFormat::Nix32, true));
 
     teeSink
         << exportMagic
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 5c413aa77..2bd3a2edc 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -50,7 +50,7 @@ static void makeSymlink(const Path & link, const Path & target)
 
 void LocalStore::addIndirectRoot(const Path & path)
 {
-    std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Base32, false);
+    std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Nix32, false);
     Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash));
     makeSymlink(realRoot, path);
 }
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index ef7dd7985..7e82bae28 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1080,7 +1080,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             if (hashResult.first != info.narHash)
                 throw Error("hash mismatch importing path '%s';\n  specified: %s\n  got:       %s",
-                    printStorePath(info.path), info.narHash.to_string(HashFormat::Base32, true), hashResult.first.to_string(HashFormat::Base32, true));
+                            printStorePath(info.path), info.narHash.to_string(HashFormat::Nix32, true), hashResult.first.to_string(HashFormat::Nix32, true));
 
             if (hashResult.second != info.narSize)
                 throw Error("size mismatch importing path '%s';\n  specified: %s\n  got:       %s",
@@ -1096,8 +1096,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
                 if (specified.hash != actualHash.hash) {
                     throw Error("ca hash mismatch importing path '%s';\n  specified: %s\n  got:       %s",
                         printStorePath(info.path),
-                        specified.hash.to_string(HashFormat::Base32, true),
-                        actualHash.hash.to_string(HashFormat::Base32, true));
+                        specified.hash.to_string(HashFormat::Nix32, true),
+                        actualHash.hash.to_string(HashFormat::Nix32, true));
                 }
             }
 
@@ -1389,7 +1389,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
         for (auto & link : readDirectory(linksDir)) {
             printMsg(lvlTalkative, "checking contents of '%s'", link.name);
             Path linkPath = linksDir + "/" + link.name;
-            std::string hash = hashPath(HashAlgorithm::SHA256, linkPath).first.to_string(HashFormat::Base32, false);
+            std::string hash = hashPath(HashAlgorithm::SHA256, linkPath).first.to_string(HashFormat::Nix32, false);
             if (hash != link.name) {
                 printError("link '%s' was modified! expected hash '%s', got '%s'",
                     linkPath, link.name, hash);
@@ -1422,7 +1422,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
                 if (info->narHash != nullHash && info->narHash != current.first) {
                     printError("path '%s' was modified! expected hash '%s', got '%s'",
-                        printStorePath(i), info->narHash.to_string(HashFormat::Base32, true), current.first.to_string(HashFormat::Base32, true));
+                               printStorePath(i), info->narHash.to_string(HashFormat::Nix32, true), current.first.to_string(HashFormat::Nix32, true));
                     if (repair) repairPath(i); else errors = true;
                 } else {
 
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index e50c15939..310105c75 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -333,9 +333,9 @@ public:
                     (std::string(info->path.name()))
                     (narInfo ? narInfo->url : "", narInfo != 0)
                     (narInfo ? narInfo->compression : "", narInfo != 0)
-                    (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(HashFormat::Base32, true) : "", narInfo && narInfo->fileHash)
+                    (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(HashFormat::Nix32, true) : "", narInfo && narInfo->fileHash)
                     (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)
-                    (info->narHash.to_string(HashFormat::Base32, true))
+                    (info->narHash.to_string(HashFormat::Nix32, true))
                     (info->narSize)
                     (concatStringsSep(" ", info->shortRefs()))
                     (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver)
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index 25e2a7d7b..d9618d04c 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -114,10 +114,10 @@ std::string NarInfo::to_string(const Store & store) const
     assert(compression != "");
     res += "Compression: " + compression + "\n";
     assert(fileHash && fileHash->algo == HashAlgorithm::SHA256);
-    res += "FileHash: " + fileHash->to_string(HashFormat::Base32, true) + "\n";
+    res += "FileHash: " + fileHash->to_string(HashFormat::Nix32, true) + "\n";
     res += "FileSize: " + std::to_string(fileSize) + "\n";
     assert(narHash.algo == HashAlgorithm::SHA256);
-    res += "NarHash: " + narHash.to_string(HashFormat::Base32, true) + "\n";
+    res += "NarHash: " + narHash.to_string(HashFormat::Nix32, true) + "\n";
     res += "NarSize: " + std::to_string(narSize) + "\n";
 
     res += "References: " + concatStringsSep(" ", shortRefs()) + "\n";
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index cadf88347..b395453d1 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -147,10 +147,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
        contents of the symlink (i.e. the result of readlink()), not
        the contents of the target (which may not even exist). */
     Hash hash = hashPath(HashAlgorithm::SHA256, path).first;
-    debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Base32, true));
+    debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true));
 
     /* Check if this is a known hash. */
-    Path linkPath = linksDir + "/" + hash.to_string(HashFormat::Base32, false);
+    Path linkPath = linksDir + "/" + hash.to_string(HashFormat::Nix32, false);
 
     /* Maybe delete the link, if it has been corrupted. */
     if (pathExists(linkPath)) {
diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc
index 73e55a96c..72f45143d 100644
--- a/src/libstore/parsed-derivations.cc
+++ b/src/libstore/parsed-derivations.cc
@@ -146,7 +146,7 @@ static nlohmann::json pathInfoToJSON(
         auto info = store.queryPathInfo(storePath);
 
         auto & jsonPath = jsonList.emplace_back(
-            info->toJSON(store, false, HashFormat::Base32));
+            info->toJSON(store, false, HashFormat::Nix32));
 
         // Add the path to the object whose metadata we are including.
         jsonPath["path"] = store.printStorePath(storePath);
diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc
index 2d7dc972f..f58e31bfd 100644
--- a/src/libstore/path-info.cc
+++ b/src/libstore/path-info.cc
@@ -31,9 +31,9 @@ std::string ValidPathInfo::fingerprint(const Store & store) const
         throw Error("cannot calculate fingerprint of path '%s' because its size is not known",
             store.printStorePath(path));
     return
-        "1;" + store.printStorePath(path) + ";"
-        + narHash.to_string(HashFormat::Base32, true) + ";"
-        + std::to_string(narSize) + ";"
+            "1;" + store.printStorePath(path) + ";"
+            + narHash.to_string(HashFormat::Nix32, true) + ";"
+            + std::to_string(narSize) + ";"
         + concatStringsSep(",", store.printStorePathSet(references));
 }
 
diff --git a/src/libstore/path.cc b/src/libstore/path.cc
index d5257c939..1afd10af7 100644
--- a/src/libstore/path.cc
+++ b/src/libstore/path.cc
@@ -35,7 +35,7 @@ StorePath::StorePath(std::string_view _baseName)
 }
 
 StorePath::StorePath(const Hash & hash, std::string_view _name)
-    : baseName((hash.to_string(HashFormat::Base32, false) + "-").append(std::string(_name)))
+    : baseName((hash.to_string(HashFormat::Nix32, false) + "-").append(std::string(_name)))
 {
     checkName(baseName, name());
 }
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 7ea1647d9..e2668c673 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -544,11 +544,45 @@ nlohmann::json Args::toJSON()
     return res;
 }
 
+static void hashFormatCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
+{
+    for (auto & format : hashFormats) {
+        if (hasPrefix(format, prefix)) {
+            completions.add(format);
+        }
+    }
+}
+
+Args::Flag Args::Flag::mkHashFormatFlagWithDefault(std::string &&longName, HashFormat * hf) {
+    assert(*hf == nix::HashFormat::SRI);
+    return Flag{
+            .longName = std::move(longName),
+            .description = "hash format ('base16', 'nix32', 'base64', 'sri'). Default: 'sri'",
+            .labels = {"hash-format"},
+            .handler = {[hf](std::string s) {
+                *hf = parseHashFormat(s);
+            }},
+            .completer = hashFormatCompleter,
+    };
+}
+
+Args::Flag Args::Flag::mkHashFormatOptFlag(std::string && longName, std::optional<HashFormat> * ohf) {
+    return Flag{
+            .longName = std::move(longName),
+            .description = "hash format ('base16', 'nix32', 'base64', 'sri').",
+            .labels = {"hash-format"},
+            .handler = {[ohf](std::string s) {
+                *ohf = std::optional<HashFormat>{parseHashFormat(s)};
+            }},
+            .completer = hashFormatCompleter,
+    };
+}
+
 static void hashAlgoCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
 {
-    for (auto & type : hashAlgorithms)
-        if (hasPrefix(type, prefix))
-            completions.add(type);
+    for (auto & algo : hashAlgorithms)
+        if (hasPrefix(algo, prefix))
+            completions.add(algo);
 }
 
 Args::Flag Args::Flag::mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha)
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 653a9bbd6..18b0ae583 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -15,6 +15,7 @@
 namespace nix {
 
 enum struct HashAlgorithm : char;
+enum struct HashFormat : int;
 
 class MultiCommand;
 
@@ -177,6 +178,8 @@ protected:
 
         static Flag mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha);
         static Flag mkHashAlgoOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha);
+        static Flag mkHashFormatFlagWithDefault(std::string && longName, HashFormat * hf);
+        static Flag mkHashFormatOptFlag(std::string && longName, std::optional<HashFormat> * ohf);
     };
 
     /**
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 38a29c459..30456ae5c 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -27,8 +27,9 @@ static size_t regularHashSize(HashAlgorithm type) {
 }
 
 
-std::set<std::string> hashAlgorithms = {"md5", "sha1", "sha256", "sha512" };
+const std::set<std::string> hashAlgorithms = {"md5", "sha1", "sha256", "sha512" };
 
+const std::set<std::string> hashFormats = {"base64", "nix32", "base16", "sri" };
 
 Hash::Hash(HashAlgorithm algo) : algo(algo)
 {
@@ -81,7 +82,7 @@ static std::string printHash16(const Hash & hash)
 
 
 // omitted: E O U T
-const std::string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz";
+const std::string nix32Chars = "0123456789abcdfghijklmnpqrsvwxyz";
 
 
 static std::string printHash32(const Hash & hash)
@@ -100,7 +101,7 @@ static std::string printHash32(const Hash & hash)
         unsigned char c =
             (hash.hash[i] >> j)
             | (i >= hash.hashSize - 1 ? 0 : hash.hash[i + 1] << (8 - j));
-        s.push_back(base32Chars[c & 0x1f]);
+        s.push_back(nix32Chars[c & 0x1f]);
     }
 
     return s;
@@ -110,7 +111,7 @@ static std::string printHash32(const Hash & hash)
 std::string printHash16or32(const Hash & hash)
 {
     assert(static_cast<char>(hash.algo));
-    return hash.to_string(hash.algo == HashAlgorithm::MD5 ? HashFormat::Base16 : HashFormat::Base32, false);
+    return hash.to_string(hash.algo == HashAlgorithm::MD5 ? HashFormat::Base16 : HashFormat::Nix32, false);
 }
 
 
@@ -125,7 +126,7 @@ std::string Hash::to_string(HashFormat hashFormat, bool includeAlgo) const
     case HashFormat::Base16:
         s += printHash16(*this);
         break;
-    case HashFormat::Base32:
+    case HashFormat::Nix32:
         s += printHash32(*this);
         break;
     case HashFormat::Base64:
@@ -230,8 +231,8 @@ Hash::Hash(std::string_view rest, HashAlgorithm algo, bool isSRI)
         for (unsigned int n = 0; n < rest.size(); ++n) {
             char c = rest[rest.size() - n - 1];
             unsigned char digit;
-            for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */
-                if (base32Chars[digit] == c) break;
+            for (digit = 0; digit < nix32Chars.size(); ++digit) /* !!! slow */
+                if (nix32Chars[digit] == c) break;
             if (digit >= 32)
                 throw BadHash("invalid base-32 hash '%s'", rest);
             unsigned int b = n * 5;
@@ -388,7 +389,11 @@ Hash compressHash(const Hash & hash, unsigned int newSize)
 std::optional<HashFormat> parseHashFormatOpt(std::string_view hashFormatName)
 {
     if (hashFormatName == "base16") return HashFormat::Base16;
-    if (hashFormatName == "base32") return HashFormat::Base32;
+    if (hashFormatName == "nix32") return HashFormat::Nix32;
+    if (hashFormatName == "base32") {
+        warn(R"("base32" is a deprecated alias for hash format "nix32".)");
+        return HashFormat::Nix32;
+    }
     if (hashFormatName == "base64") return HashFormat::Base64;
     if (hashFormatName == "sri") return HashFormat::SRI;
     return std::nullopt;
@@ -407,8 +412,8 @@ std::string_view printHashFormat(HashFormat HashFormat)
     switch (HashFormat) {
     case HashFormat::Base64:
         return "base64";
-    case HashFormat::Base32:
-        return "base32";
+    case HashFormat::Nix32:
+        return "nix32";
     case HashFormat::Base16:
         return "base16";
     case HashFormat::SRI:
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index 3c97ed4b1..7bed9e2bd 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -20,9 +20,9 @@ const int sha1HashSize = 20;
 const int sha256HashSize = 32;
 const int sha512HashSize = 64;
 
-extern std::set<std::string> hashAlgorithms;
+extern const std::set<std::string> hashAlgorithms;
 
-extern const std::string base32Chars;
+extern const std::string nix32Chars;
 
 /**
  * @brief Enumeration representing the hash formats.
@@ -31,8 +31,8 @@ enum struct HashFormat : int {
     /// @brief Base 64 encoding.
     /// @see [IETF RFC 4648, section 4](https://datatracker.ietf.org/doc/html/rfc4648#section-4).
     Base64,
-    /// @brief Nix-specific base-32 encoding. @see base32Chars
-    Base32,
+    /// @brief Nix-specific base-32 encoding. @see nix32Chars
+    Nix32,
     /// @brief Lowercase hexadecimal encoding. @see base16Chars
     Base16,
     /// @brief "<hash algo>:<Base 64 hash>", format of the SRI integrity attribute.
@@ -40,6 +40,8 @@ enum struct HashFormat : int {
     SRI
 };
 
+extern const std::set<std::string> hashFormats;
+
 struct Hash
 {
     constexpr static size_t maxHashSize = 64;
diff --git a/src/libutil/references.cc b/src/libutil/references.cc
index d82d51945..b30e62c7b 100644
--- a/src/libutil/references.cc
+++ b/src/libutil/references.cc
@@ -23,8 +23,8 @@ static void search(
     static bool isBase32[256];
     std::call_once(initialised, [](){
         for (unsigned int i = 0; i < 256; ++i) isBase32[i] = false;
-        for (unsigned int i = 0; i < base32Chars.size(); ++i)
-            isBase32[(unsigned char) base32Chars[i]] = true;
+        for (unsigned int i = 0; i < nix32Chars.size(); ++i)
+            isBase32[(unsigned char) nix32Chars[i]] = true;
     });
 
     for (size_t i = 0; i + refLength <= s.size(); ) {
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 75ad4e75f..db45be2a8 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -406,7 +406,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
                     auto info = store->queryPathInfo(j);
                     if (query == qHash) {
                         assert(info->narHash.algo == HashAlgorithm::SHA256);
-                        cout << fmt("%s\n", info->narHash.to_string(HashFormat::Base32, true));
+                        cout << fmt("%s\n", info->narHash.to_string(HashFormat::Nix32, true));
                     } else if (query == qSize)
                         cout << fmt("%d\n", info->narSize);
                 }
@@ -769,8 +769,8 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
         if (current.first != info->narHash) {
             printError("path '%s' was modified! expected hash '%s', got '%s'",
                 store->printStorePath(path),
-                info->narHash.to_string(HashFormat::Base32, true),
-                current.first.to_string(HashFormat::Base32, true));
+                info->narHash.to_string(HashFormat::Nix32, true),
+                current.first.to_string(HashFormat::Nix32, true));
             status = 1;
         }
     }
@@ -898,7 +898,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                         out << info->narSize // downloadSize
                             << info->narSize;
                         if (GET_PROTOCOL_MINOR(clientVersion) >= 4)
-                            out << info->narHash.to_string(HashFormat::Base32, true)
+                            out << info->narHash.to_string(HashFormat::Nix32, true)
                                 << renderContentAddress(info->ca)
                                 << info->sigs;
                     } catch (InvalidPath &) {
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 173043c8a..f9c7592a3 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -39,7 +39,7 @@ struct CmdHashBase : Command
         addFlag({
             .longName = "base32",
             .description = "Print the hash in base-32 (Nix-specific) format.",
-            .handler = {&hashFormat, HashFormat::Base32},
+            .handler = {&hashFormat, HashFormat::Nix32},
         });
 
         addFlag({
@@ -120,7 +120,7 @@ struct CmdToBase : Command
     {
         return fmt("convert a hash to %s representation",
             hashFormat == HashFormat::Base16 ? "base-16" :
-            hashFormat == HashFormat::Base32 ? "base-32" :
+            hashFormat == HashFormat::Nix32 ? "base-32" :
             hashFormat == HashFormat::Base64 ? "base-64" :
             "SRI");
     }
@@ -143,24 +143,8 @@ struct CmdHashConvert : Command
     std::vector<std::string> hashStrings;
 
     CmdHashConvert(): to(HashFormat::SRI) {
-        addFlag({
-            .longName = "from",
-            // TODO: List format choices. Maybe introduce a constant?
-            .description = "The format of the input hash.",
-            .labels = {"hash format"},
-            .handler = {[this](std::string str) {
-                from = parseHashFormat(str);
-            }},
-        });
-        addFlag({
-            .longName = "to",
-            // TODO: List format choices. Maybe introduce a constant?
-            .description = "The format of the output hash.",
-            .labels = {"hash format"},
-            .handler = {[this](std::string str) {
-                to = parseHashFormat(str);
-            }},
-        });
+        addFlag(Args::Flag::mkHashFormatOptFlag("from", &from));
+        addFlag(Args::Flag::mkHashFormatFlagWithDefault("to", &to));
         addFlag(Args::Flag::mkHashAlgoOptFlag("algo", &algo));
         expectArgs({
            .label = "hashes",
@@ -170,7 +154,15 @@ struct CmdHashConvert : Command
 
     std::string description() override
     {
-        return "convert between different hash formats, e.g. base16, nix32, base64 and sri.";
+        std::string descr( "convert between different hash formats. Choose from: ");
+        auto iter = hashFormats.begin();
+        assert(iter != hashFormats.end());
+        descr += *iter++;
+        while (iter != hashFormats.end()) {
+            descr += ", " + *iter++;
+        }
+
+        return descr;
     }
 
     Category category() override { return catUtility; }
@@ -197,7 +189,7 @@ struct CmdHash : NixMultiCommand
                 {"file", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Flat);; }},
                 {"path", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Recursive); }},
                 {"to-base16", []() { return make_ref<CmdToBase>(HashFormat::Base16); }},
-                {"to-base32", []() { return make_ref<CmdToBase>(HashFormat::Base32); }},
+                {"to-base32", []() { return make_ref<CmdToBase>(HashFormat::Nix32); }},
                 {"to-base64", []() { return make_ref<CmdToBase>(HashFormat::Base64); }},
                 {"to-sri", []() { return make_ref<CmdToBase>(HashFormat::SRI); }},
           })
@@ -230,7 +222,7 @@ static int compatNixHash(int argc, char * * argv)
             printVersion("nix-hash");
         else if (*arg == "--flat") flat = true;
         else if (*arg == "--base16") hashFormat = HashFormat::Base16;
-        else if (*arg == "--base32") hashFormat = HashFormat::Base32;
+        else if (*arg == "--base32") hashFormat = HashFormat::Nix32;
         else if (*arg == "--base64") hashFormat = HashFormat::Base64;
         else if (*arg == "--sri") hashFormat = HashFormat::SRI;
         else if (*arg == "--truncate") truncate = true;
@@ -244,7 +236,7 @@ static int compatNixHash(int argc, char * * argv)
         }
         else if (*arg == "--to-base32") {
             op = opTo;
-            hashFormat = HashFormat::Base32;
+            hashFormat = HashFormat::Nix32;
         }
         else if (*arg == "--to-base64") {
             op = opTo;
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index cd0f6d95f..f0234f7be 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -109,8 +109,8 @@ struct CmdVerify : StorePathsCommand
                         act2.result(resCorruptedPath, store->printStorePath(info->path));
                         printError("path '%s' was modified! expected hash '%s', got '%s'",
                             store->printStorePath(info->path),
-                            info->narHash.to_string(HashFormat::Base32, true),
-                            hash.first.to_string(HashFormat::Base32, true));
+                            info->narHash.to_string(HashFormat::Nix32, true),
+                            hash.first.to_string(HashFormat::Nix32, true));
                     }
                 }
 
diff --git a/tests/functional/hash.sh b/tests/functional/hash.sh
index 031e33adf..278ed83b9 100644
--- a/tests/functional/hash.sh
+++ b/tests/functional/hash.sh
@@ -163,7 +163,7 @@ try3() {
 
     sri=$(nix hash convert --algo "$1" --from base16 "$2")
     [ "$sri" = "$1-$4" ]
-    sri=$(nix hash convert --algo "$1" --from base32 "$3")
+    sri=$(nix hash convert --algo "$1" --from nix32 "$3")
     [ "$sri" = "$1-$4" ]
     sri=$(nix hash convert --algo "$1" --from base64 "$4")
     [ "$sri" = "$1-$4" ]
@@ -172,11 +172,11 @@ try3() {
     # Asserting input format fails.
     #
 
-    fail=$(nix hash convert --algo "$1" --from base32 "$2" 2>&1 || echo "exit: $?")
+    fail=$(nix hash convert --algo "$1" --from nix32 "$2" 2>&1 || echo "exit: $?")
     [[ "$fail" == "error: input hash"*"exit: 1" ]]
     fail=$(nix hash convert --algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?")
     [[ "$fail" == "error: input hash"*"exit: 1" ]]
-    fail=$(nix hash convert --algo "$1" --from base32 "$4" 2>&1 || echo "exit: $?")
+    fail=$(nix hash convert --algo "$1" --from nix32 "$4" 2>&1 || echo "exit: $?")
     [[ "$fail" == "error: input hash"*"exit: 1" ]]
 
 }
diff --git a/tests/functional/lang/eval-okay-convertHash.err.exp b/tests/functional/lang/eval-okay-convertHash.err.exp
new file mode 100644
index 000000000..41d746725
--- /dev/null
+++ b/tests/functional/lang/eval-okay-convertHash.err.exp
@@ -0,0 +1,108 @@
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
+warning: "base32" is a deprecated alias for hash format "nix32".
diff --git a/tests/functional/lang/eval-okay-convertHash.exp b/tests/functional/lang/eval-okay-convertHash.exp
index 60e0a3c49..16b0240e5 100644
--- a/tests/functional/lang/eval-okay-convertHash.exp
+++ b/tests/functional/lang/eval-okay-convertHash.exp
@@ -1 +1 @@
-{ hashesBase16 = [ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ]; hashesBase32 = [ "3y8bwfr609h3lh9ch0izcqq7fl" "26mrvc0v1nslch8r0w45zywsbc" "1v4gi57l97pmnylq6lmgxkhd5v" "143xibwh31h9bvxzalr0sjvbbvpa6ffs" "i4hj30pkrfdpgc5dbcgcydqviibfhm6d" "fxz2p030yba2bza71qhss79k3l5y24kd" "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" "0qy6iz9yh6a079757mxdmypx0gcmnzjd3ij5q78bzk00vxll82lh" "0mkygpci4r4yb8zz5rs2kxcgvw0a2yf5zlj6r8qgfll6pnrqf0xd" "0zdl9zrg8r3i9c1g90lgg9ip5ijzv3yhz91i0zzn3r8ap9ws784gkp9dk9j3aglhgf1amqb0pj21mh7h1nxcl18akqvvf7ggqsy30yg" "19ncrpp37dx0nzzjw4k6zaqkb9mzaq2myhgpzh5aff7qqcj5wwdxslg6ixwncm7gyq8l761gwf87fgsh2bwfyr52s53k2dkqvw8c24x" "2kz74snvckxldmmbisz9ikmy031d28cs6xfdbl6rhxx42glpyz4vww4lajrc5akklxwixl0js4g84233pxvmbykiic5m7i5m9r4nr11" ]; hashesBase64 = [ "1B2M2Y8AsgTpgAmY7PhCfg==" "bGnufyEcZAQZ1TZswHauRg==" "uzQ4+6vUYOptvSfRU+IjOw==" "2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "zVToVowbN88eW62wd5vL84IhIYk=" "bRLhCx0zHa0hDkf9JdTyYIArfnc=" "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; hashesSRI = [ "md5-1B2M2Y8AsgTpgAmY7PhCfg==" "md5-bGnufyEcZAQZ1TZswHauRg==" "md5-uzQ4+6vUYOptvSfRU+IjOw==" "sha1-2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "sha1-zVToVowbN88eW62wd5vL84IhIYk=" "sha1-bRLhCx0zHa0hDkf9JdTyYIArfnc=" "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "sha256-kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "sha256-rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "sha512-nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "sha512-IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; }
+{ hashesBase16 = [ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ]; hashesBase32 = [ "3y8bwfr609h3lh9ch0izcqq7fl" "26mrvc0v1nslch8r0w45zywsbc" "1v4gi57l97pmnylq6lmgxkhd5v" "143xibwh31h9bvxzalr0sjvbbvpa6ffs" "i4hj30pkrfdpgc5dbcgcydqviibfhm6d" "fxz2p030yba2bza71qhss79k3l5y24kd" "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" "0qy6iz9yh6a079757mxdmypx0gcmnzjd3ij5q78bzk00vxll82lh" "0mkygpci4r4yb8zz5rs2kxcgvw0a2yf5zlj6r8qgfll6pnrqf0xd" "0zdl9zrg8r3i9c1g90lgg9ip5ijzv3yhz91i0zzn3r8ap9ws784gkp9dk9j3aglhgf1amqb0pj21mh7h1nxcl18akqvvf7ggqsy30yg" "19ncrpp37dx0nzzjw4k6zaqkb9mzaq2myhgpzh5aff7qqcj5wwdxslg6ixwncm7gyq8l761gwf87fgsh2bwfyr52s53k2dkqvw8c24x" "2kz74snvckxldmmbisz9ikmy031d28cs6xfdbl6rhxx42glpyz4vww4lajrc5akklxwixl0js4g84233pxvmbykiic5m7i5m9r4nr11" ]; hashesBase64 = [ "1B2M2Y8AsgTpgAmY7PhCfg==" "bGnufyEcZAQZ1TZswHauRg==" "uzQ4+6vUYOptvSfRU+IjOw==" "2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "zVToVowbN88eW62wd5vL84IhIYk=" "bRLhCx0zHa0hDkf9JdTyYIArfnc=" "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; hashesNix32 = [ "3y8bwfr609h3lh9ch0izcqq7fl" "26mrvc0v1nslch8r0w45zywsbc" "1v4gi57l97pmnylq6lmgxkhd5v" "143xibwh31h9bvxzalr0sjvbbvpa6ffs" "i4hj30pkrfdpgc5dbcgcydqviibfhm6d" "fxz2p030yba2bza71qhss79k3l5y24kd" "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" "0qy6iz9yh6a079757mxdmypx0gcmnzjd3ij5q78bzk00vxll82lh" "0mkygpci4r4yb8zz5rs2kxcgvw0a2yf5zlj6r8qgfll6pnrqf0xd" "0zdl9zrg8r3i9c1g90lgg9ip5ijzv3yhz91i0zzn3r8ap9ws784gkp9dk9j3aglhgf1amqb0pj21mh7h1nxcl18akqvvf7ggqsy30yg" "19ncrpp37dx0nzzjw4k6zaqkb9mzaq2myhgpzh5aff7qqcj5wwdxslg6ixwncm7gyq8l761gwf87fgsh2bwfyr52s53k2dkqvw8c24x" "2kz74snvckxldmmbisz9ikmy031d28cs6xfdbl6rhxx42glpyz4vww4lajrc5akklxwixl0js4g84233pxvmbykiic5m7i5m9r4nr11" ]; hashesSRI = [ "md5-1B2M2Y8AsgTpgAmY7PhCfg==" "md5-bGnufyEcZAQZ1TZswHauRg==" "md5-uzQ4+6vUYOptvSfRU+IjOw==" "sha1-2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "sha1-zVToVowbN88eW62wd5vL84IhIYk=" "sha1-bRLhCx0zHa0hDkf9JdTyYIArfnc=" "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "sha256-kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "sha256-rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "sha512-nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "sha512-IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; }
diff --git a/tests/functional/lang/eval-okay-convertHash.nix b/tests/functional/lang/eval-okay-convertHash.nix
index cf4909aaf..a0191ee8d 100644
--- a/tests/functional/lang/eval-okay-convertHash.nix
+++ b/tests/functional/lang/eval-okay-convertHash.nix
@@ -5,12 +5,14 @@ let
   map2' = f: fsts: snds: map2 f { inherit fsts snds; };
   getOutputHashes = hashes: {
     hashesBase16 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base16";}) hashAlgos hashes;
+    hashesNix32 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";}) hashAlgos hashes;
     hashesBase32 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";}) hashAlgos hashes;
     hashesBase64 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base64";}) hashAlgos hashes;
     hashesSRI    = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "sri"   ;}) hashAlgos hashes;
   };
   getOutputHashesColon = hashes: {
     hashesBase16 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base16";}) hashAlgos hashes;
+    hashesNix32 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "nix32";}) hashAlgos hashes;
     hashesBase32 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base32";}) hashAlgos hashes;
     hashesBase64 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base64";}) hashAlgos hashes;
     hashesSRI    = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "sri"   ;}) hashAlgos hashes;
diff --git a/tests/unit/libutil/hash.cc b/tests/unit/libutil/hash.cc
index 4d82c7f09..a88994d0b 100644
--- a/tests/unit/libutil/hash.cc
+++ b/tests/unit/libutil/hash.cc
@@ -80,7 +80,7 @@ namespace nix {
      * --------------------------------------------------------------------------*/
 
     TEST(hashFormat, testRoundTripPrintParse) {
-        for (const HashFormat hashFormat: { HashFormat::Base64, HashFormat::Base32, HashFormat::Base16, HashFormat::SRI}) {
+        for (const HashFormat hashFormat: { HashFormat::Base64, HashFormat::Nix32, HashFormat::Base16, HashFormat::SRI}) {
             ASSERT_EQ(parseHashFormat(printHashFormat(hashFormat)), hashFormat);
             ASSERT_EQ(*parseHashFormatOpt(printHashFormat(hashFormat)), hashFormat);
         }

From 7ff876b92b590fd9559472935f4adce1d3d5efb7 Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Tue, 28 Nov 2023 15:58:04 +0100
Subject: [PATCH 178/421] Add deprecation notice for old nix hash conversion
 subcommands.

(But not yet nix-hash since `nix hash` is still hidden behind a feature flag.)

https://github.com/NixOS/nix/issues/8876
---
 src/nix/hash.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index f9c7592a3..2c9deb0d5 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -127,6 +127,7 @@ struct CmdToBase : Command
 
     void run() override
     {
+        warn("The old format conversion sub commands of `nix hash` where deprecated in favor of `nix hash convert`.");
         for (auto s : args)
             logger->cout(Hash::parseAny(s, ht).to_string(hashFormat, hashFormat == HashFormat::SRI));
     }
@@ -208,6 +209,9 @@ static auto rCmdHash = registerCommand<CmdHash>("hash");
 /* Legacy nix-hash command. */
 static int compatNixHash(int argc, char * * argv)
 {
+    // Wait until `nix hash convert` is not hidden behind experimental flags anymore.
+    // warn("`nix-hash` has been deprecated in favor of `nix hash convert`.");
+
     std::optional<HashAlgorithm> ha;
     bool flat = false;
     HashFormat hashFormat = HashFormat::Base16;

From 8afeaf05c4063d48e65d2d82c31c3323c3237f7c Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Tue, 28 Nov 2023 19:02:15 +0100
Subject: [PATCH 179/421] Add docs/rl-notes for `nix hash convert` /
 `builtins.convertHash`

https://github.com/NixOS/nix/issues/8876
---
 doc/manual/rl-next/hash-format-nix32.md | 22 ++++++++++++
 doc/manual/rl-next/nix-hash-convert.md  | 47 +++++++++++++++++++++++++
 src/libexpr/primops.cc                  |  8 ++---
 src/nix/hash.cc                         |  2 +-
 4 files changed, 74 insertions(+), 5 deletions(-)
 create mode 100644 doc/manual/rl-next/hash-format-nix32.md
 create mode 100644 doc/manual/rl-next/nix-hash-convert.md

diff --git a/doc/manual/rl-next/hash-format-nix32.md b/doc/manual/rl-next/hash-format-nix32.md
new file mode 100644
index 000000000..20c557da9
--- /dev/null
+++ b/doc/manual/rl-next/hash-format-nix32.md
@@ -0,0 +1,22 @@
+synopsis: Rename hash format `base32` to `nix32`
+prs: #9452
+description: {
+
+Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for 
+[Base32](https://en.wikipedia.org/wiki/Base32).
+
+## Deprecation: Use `nix32` instead of `base32` as `toHashFormat`
+
+For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from`
+parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value
+remains as a deprecated alias for `"base32"`. Please convert your code from:
+
+```nix
+builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";}
+```
+
+to
+
+```nix
+builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";}
+```
\ No newline at end of file
diff --git a/doc/manual/rl-next/nix-hash-convert.md b/doc/manual/rl-next/nix-hash-convert.md
new file mode 100644
index 000000000..de4367c5b
--- /dev/null
+++ b/doc/manual/rl-next/nix-hash-convert.md
@@ -0,0 +1,47 @@
+synopsis: Add `nix hash convert`
+prs: #9452
+description: {
+
+New [`nix hash convert`](https://github.com/NixOS/nix/issues/8876) sub command with a fast track
+to stabilization! Examples:
+
+- Convert the hash to `nix32`.
+
+  ```bash
+  $ nix hash convert --algo "sha1" --to nix32 "800d59cfcd3c05e900cb4e214be48f6b886a08df"
+  vw46m23bizj4n8afrc0fj19wrp7mj3c0
+  ```
+  `nix32` is a base32 encoding with a nix-specific character set.
+  Explicitly specify the hashing algorithm (optional with SRI hashes) but detect hash format by the length of the input
+  hash.
+- Convert the hash to the `sri` format that includes an algorithm specification:
+  ```bash
+  nix hash convert --algo "sha1" "800d59cfcd3c05e900cb4e214be48f6b886a08df"
+  sha1-gA1Zz808BekAy04hS+SPa4hqCN8=
+  ```
+  or with an explicit `-to` format:
+  ```bash
+  nix hash convert --algo "sha1" --to sri "800d59cfcd3c05e900cb4e214be48f6b886a08df"
+  sha1-gA1Zz808BekAy04hS+SPa4hqCN8=
+  ```
+- Assert the input format of the hash:
+  ```bash
+  nix hash convert --algo "sha256" --from nix32 "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="
+  error: input hash 'ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=' does not have the expected format '--from nix32'
+  nix hash convert --algo "sha256" --from nix32 "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s"
+  sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=
+  ```
+
+The `--to`/`--from`/`--algo` parameters have context-sensitive auto-completion.
+
+## Related Deprecations
+
+The following commands are still available but will emit a deprecation warning. Please convert your code to
+`nix hash convert`:
+
+- `nix hash to-base16 $hash1 $hash2`: Use `nix hash convert --to base16 $hash1 $hash2` instead.
+- `nix hash to-base32 $hash1 $hash2`: Use `nix hash convert --to nix32 $hash1 $hash2` instead.
+- `nix hash to-base64 $hash1 $hash2`: Use `nix hash convert --to base64 $hash1 $hash2` instead.
+- `nix hash to-sri $hash1 $hash2`: : Use `nix hash convert --to sri $hash1 $hash2`
+  or even just `nix hash convert $hash1 $hash2` instead.
+}
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 7831f3803..4162a8da3 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1339,7 +1339,7 @@ drvName, Bindings * attrs, Value & v)
                 .errPos = state.positions[noPos]
             });
 
-        auto ht = parseHashAlgoOpt(outputHashAlgo).value_or(HashAlgorithm::SHA256);
+        auto ha = parseHashAlgoOpt(outputHashAlgo).value_or(HashAlgorithm::SHA256);
         auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive);
 
         for (auto & i : outputs) {
@@ -1348,13 +1348,13 @@ drvName, Bindings * attrs, Value & v)
                 drv.outputs.insert_or_assign(i,
                     DerivationOutput::Impure {
                         .method = method,
-                        .hashAlgo = ht,
+                        .hashAlgo = ha,
                     });
             else
                 drv.outputs.insert_or_assign(i,
                     DerivationOutput::CAFloating {
                         .method = method,
-                        .hashAlgo = ht,
+                        .hashAlgo = ha,
                     });
         }
     }
@@ -3837,7 +3837,7 @@ static RegisterPrimOp primop_convertHash({
 
         The format of the resulting hash. Must be one of
         - `"base16"`
-        - `"base32"`
+        - `"nix32"`
         - `"base64"`
         - `"sri"`
 
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 2c9deb0d5..0bba3b7d2 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -118,7 +118,7 @@ struct CmdToBase : Command
 
     std::string description() override
     {
-        return fmt("convert a hash to %s representation",
+        return fmt("convert a hash to %s representation (deprecated, use `nix hash convert` instead)",
             hashFormat == HashFormat::Base16 ? "base-16" :
             hashFormat == HashFormat::Nix32 ? "base-32" :
             hashFormat == HashFormat::Base64 ? "base-64" :

From d38ec1285573c98c987ec1421f7cec68754204f9 Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Sat, 2 Dec 2023 11:53:50 +0100
Subject: [PATCH 180/421] Update src/libexpr/primops.cc
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com>
---
 src/libexpr/primops.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 4162a8da3..828d118eb 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -3838,6 +3838,7 @@ static RegisterPrimOp primop_convertHash({
         The format of the resulting hash. Must be one of
         - `"base16"`
         - `"nix32"`
+        - `"base32"` (deprecated alias for `"nix32"`)
         - `"base64"`
         - `"sri"`
 

From bbba2055f0b77e9677ef318ceea3084906eccd7d Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Sat, 2 Dec 2023 16:43:52 +0100
Subject: [PATCH 181/421] Refactor concurrently added tests to use
 HashAlgorithm.

https://github.com/NixOS/nix/issues/8876
---
 tests/unit/libutil-support/tests/hash.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unit/libutil-support/tests/hash.cc b/tests/unit/libutil-support/tests/hash.cc
index 577e9890e..50889cd33 100644
--- a/tests/unit/libutil-support/tests/hash.cc
+++ b/tests/unit/libutil-support/tests/hash.cc
@@ -11,7 +11,7 @@ using namespace nix;
 
 Gen<Hash> Arbitrary<Hash>::arbitrary()
 {
-    Hash hash(htSHA1);
+    Hash hash(HashAlgorithm::SHA1);
     for (size_t i = 0; i < hash.hashSize; ++i)
         hash.hash[i] = *gen::arbitrary<uint8_t>();
     return gen::just(hash);

From e9a5365db66737d1438fd91eba6529d278e1efca Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Sat, 2 Dec 2023 18:19:51 +0100
Subject: [PATCH 182/421] hash.sh: Make failure tests more tolerant of
 additional output

"warning: you don'\''t have Internet access; disabling some network-dependent features" ...

https://github.com/NixOS/nix/issues/8876
---
 tests/functional/hash.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/functional/hash.sh b/tests/functional/hash.sh
index 278ed83b9..47eed5178 100644
--- a/tests/functional/hash.sh
+++ b/tests/functional/hash.sh
@@ -173,11 +173,11 @@ try3() {
     #
 
     fail=$(nix hash convert --algo "$1" --from nix32 "$2" 2>&1 || echo "exit: $?")
-    [[ "$fail" == "error: input hash"*"exit: 1" ]]
+    [[ "$fail" == *"error: input hash"*"exit: 1" ]]
     fail=$(nix hash convert --algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?")
-    [[ "$fail" == "error: input hash"*"exit: 1" ]]
+    [[ "$fail" == *"error: input hash"*"exit: 1" ]]
     fail=$(nix hash convert --algo "$1" --from nix32 "$4" 2>&1 || echo "exit: $?")
-    [[ "$fail" == "error: input hash"*"exit: 1" ]]
+    [[ "$fail" == *"error: input hash"*"exit: 1" ]]
 
 }
 

From 9a1a3c43bf11912ad32c433219c4c21a1b6ca9dd Mon Sep 17 00:00:00 2001
From: Peter Kolloch <info@eigenvalue.net>
Date: Sun, 3 Dec 2023 09:50:44 +0100
Subject: [PATCH 183/421] Store.xs: fix references to HashFormat::Nix32

https://github.com/NixOS/nix/issues/8876
---
 perl/lib/Nix/Store.xs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index 50148141b..82c7db608 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -78,7 +78,7 @@ SV * queryReferences(char * path)
 SV * queryPathHash(char * path)
     PPCODE:
         try {
-            auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(HashFormat::Base32, true);
+            auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(HashFormat::Nix32, true);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -104,7 +104,7 @@ SV * queryPathInfo(char * path, int base32)
                 XPUSHs(&PL_sv_undef);
             else
                 XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
-            auto s = info->narHash.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, true);
+            auto s = info->narHash.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, true);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
             mXPUSHi(info->registrationTime);
             mXPUSHi(info->narSize);
@@ -206,7 +206,7 @@ SV * hashPath(char * algo, int base32, char * path)
     PPCODE:
         try {
             Hash h = hashPath(parseHashAlgo(algo), path).first;
-            auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false);
+            auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -217,7 +217,7 @@ SV * hashFile(char * algo, int base32, char * path)
     PPCODE:
         try {
             Hash h = hashFile(parseHashAlgo(algo), path);
-            auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false);
+            auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -228,7 +228,7 @@ SV * hashString(char * algo, int base32, char * s)
     PPCODE:
         try {
             Hash h = hashString(parseHashAlgo(algo), s);
-            auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false);
+            auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -239,7 +239,7 @@ SV * convertHash(char * algo, char * s, int toBase32)
     PPCODE:
         try {
             auto h = Hash::parseAny(s, parseHashAlgo(algo));
-            auto s = h.to_string(toBase32 ? HashFormat::Base32 : HashFormat::Base16, false);
+            auto s = h.to_string(toBase32 ? HashFormat::Nix32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());

From bf00d5ecef20c11eb7e49dff3482b9e536cf7abe Mon Sep 17 00:00:00 2001
From: Bryan Honof <bryanhonof@gmail.com>
Date: Thu, 7 Dec 2023 11:04:48 +0100
Subject: [PATCH 184/421] fix(libutil/tarfile): add option to libarchive so it
 behaves correctly with AppleDouble files

AppleDouble files were extracted differently on macOS machines than on other
UNIX's.
Setting `archive_read_set_format_option(this->archive, NULL ,"mac-ext",NULL)`
fixes this problem, since it just ignores the AppleDouble file and treats it as
a normal one.
This was a problem since it caused source archives to be different between macOS
and Linux.

Ref: nixos/nix#9290
---
 src/libutil/tarfile.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index 1733c791c..187b3e948 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -53,6 +53,7 @@ TarArchive::TarArchive(Source & source, bool raw) : buffer(65536)
         archive_read_support_format_raw(archive);
         archive_read_support_format_empty(archive);
     }
+    archive_read_set_option(archive, NULL, "mac-ext", NULL);
     check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)");
 }
 
@@ -63,6 +64,7 @@ TarArchive::TarArchive(const Path & path)
 
     archive_read_support_filter_all(archive);
     archive_read_support_format_all(archive);
+    archive_read_set_option(archive, NULL, "mac-ext", NULL);
     check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
 }
 

From a5521b7d9445af63a159d4fe7b44a0902c3a2a24 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 7 Dec 2023 10:49:29 -0500
Subject: [PATCH 185/421] Factor out
 `ServeProto::Serialiser<UnkeyedValidPathInfo>` and test

In the process, partially undo e89b5bd0bfeb4dfdd8fe7e6929544cb9ceb8a505
in that the ancient < 2.4 version is now supported again by the
serializer again. `LegacySSHStore`, instead of also asserting that the
version is at least 4, just checks that `narHash` is set.

This allows us to better test the serializer in isolation for both
versions (< 4 and >= 4).
---
 src/libstore/legacy-ssh-store.cc              |  22 ++---
 src/libstore/serve-protocol.cc                |  44 ++++++++++
 src/libstore/serve-protocol.hh                |   3 +
 src/nix-store/nix-store.cc                    |  12 +--
 .../unkeyed-valid-path-info-2.3.bin           | Bin 0 -> 184 bytes
 .../unkeyed-valid-path-info-2.4.bin           | Bin 0 -> 648 bytes
 tests/unit/libstore/serve-protocol.cc         |  77 ++++++++++++++++++
 7 files changed, 131 insertions(+), 27 deletions(-)
 create mode 100644 tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.3.bin
 create mode 100644 tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.4.bin

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index fb1580dd6..277445ee6 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -172,24 +172,12 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
             if (p.empty()) return callback(nullptr);
             auto path2 = parseStorePath(p);
             assert(path == path2);
-            /* Hash will be set below. FIXME construct ValidPathInfo at end. */
-            auto info = std::make_shared<ValidPathInfo>(path, Hash::dummy);
+            auto info = std::make_shared<ValidPathInfo>(
+                path,
+                ServeProto::Serialise<UnkeyedValidPathInfo>::read(*this, *conn));
 
-            auto deriver = readString(conn->from);
-            if (deriver != "")
-                info->deriver = parseStorePath(deriver);
-            info->references = ServeProto::Serialise<StorePathSet>::read(*this, *conn);
-            readLongLong(conn->from); // download size
-            info->narSize = readLongLong(conn->from);
-
-            {
-                auto s = readString(conn->from);
-                if (s == "")
-                    throw Error("NAR hash is now mandatory");
-                info->narHash = Hash::parseAnyPrefixed(s);
-            }
-            info->ca = ContentAddress::parseOpt(readString(conn->from));
-            info->sigs = readStrings<StringSet>(conn->from);
+            if (info->narHash == Hash::dummy)
+                throw Error("NAR hash is now mandatory");
 
             auto s = readString(conn->from);
             assert(s == "");
diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc
index fb33553c5..c37b3095c 100644
--- a/src/libstore/serve-protocol.cc
+++ b/src/libstore/serve-protocol.cc
@@ -5,6 +5,7 @@
 #include "serve-protocol.hh"
 #include "serve-protocol-impl.hh"
 #include "archive.hh"
+#include "path-info.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -54,4 +55,47 @@ void ServeProto::Serialise<BuildResult>::write(const StoreDirConfig & store, Ser
     }
 }
 
+
+UnkeyedValidPathInfo ServeProto::Serialise<UnkeyedValidPathInfo>::read(const StoreDirConfig & store, ReadConn conn)
+{
+    /* Hash should be set below unless very old `nix-store --serve`.
+       Caller should assert that it did set it. */
+    UnkeyedValidPathInfo info { Hash::dummy };
+
+    auto deriver = readString(conn.from);
+    if (deriver != "")
+        info.deriver = store.parseStorePath(deriver);
+    info.references = ServeProto::Serialise<StorePathSet>::read(store, conn);
+
+    readLongLong(conn.from); // download size, unused
+    info.narSize = readLongLong(conn.from);
+
+    if (GET_PROTOCOL_MINOR(conn.version) >= 4) {
+        auto s = readString(conn.from);
+        if (!s.empty())
+            info.narHash = Hash::parseAnyPrefixed(s);
+        info.ca = ContentAddress::parseOpt(readString(conn.from));
+        info.sigs = readStrings<StringSet>(conn.from);
+    }
+
+    return info;
+}
+
+void ServeProto::Serialise<UnkeyedValidPathInfo>::write(const StoreDirConfig & store, WriteConn conn, const UnkeyedValidPathInfo & info)
+{
+    conn.to
+        << (info.deriver ? store.printStorePath(*info.deriver) : "");
+
+    ServeProto::write(store, conn, info.references);
+    // !!! Maybe we want compression?
+    conn.to
+        << info.narSize // downloadSize, lie a little
+        << info.narSize;
+    if (GET_PROTOCOL_MINOR(conn.version) >= 4)
+        conn.to
+            << info.narHash.to_string(HashFormat::Nix32, true)
+            << renderContentAddress(info.ca)
+            << info.sigs;
+}
+
 }
diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh
index 6e9d66e2d..ada67a149 100644
--- a/src/libstore/serve-protocol.hh
+++ b/src/libstore/serve-protocol.hh
@@ -18,6 +18,7 @@ struct Source;
 
 // items being serialised
 struct BuildResult;
+struct UnkeyedValidPathInfo;
 
 
 /**
@@ -141,6 +142,8 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op)
 
 template<>
 DECLARE_SERVE_SERIALISER(BuildResult);
+template<>
+DECLARE_SERVE_SERIALISER(UnkeyedValidPathInfo);
 
 template<typename T>
 DECLARE_SERVE_SERIALISER(std::vector<T>);
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index db45be2a8..45af7879c 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -891,16 +891,8 @@ static void opServe(Strings opFlags, Strings opArgs)
                 for (auto & i : paths) {
                     try {
                         auto info = store->queryPathInfo(i);
-                        out << store->printStorePath(info->path)
-                            << (info->deriver ? store->printStorePath(*info->deriver) : "");
-                        ServeProto::write(*store, wconn, info->references);
-                        // !!! Maybe we want compression?
-                        out << info->narSize // downloadSize
-                            << info->narSize;
-                        if (GET_PROTOCOL_MINOR(clientVersion) >= 4)
-                            out << info->narHash.to_string(HashFormat::Nix32, true)
-                                << renderContentAddress(info->ca)
-                                << info->sigs;
+                        out << store->printStorePath(info->path);
+                        ServeProto::write(*store, wconn, static_cast<const UnkeyedValidPathInfo &>(*info));
                     } catch (InvalidPath &) {
                     }
                 }
diff --git a/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.3.bin b/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.3.bin
new file mode 100644
index 0000000000000000000000000000000000000000..8056ec055ed2039814ab2654aad984f07115518f
GIT binary patch
literal 184
zcmZQzKm~Rk5I&4HhDz(_Wmf1Hm*f|v>Zco)n`cxS7viFIlM;*cQi{sJIvJt*ahg+E
XS&435ArJt~*G<dMhnoX)4~zx?9JnZs

literal 0
HcmV?d00001

diff --git a/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.4.bin b/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.4.bin
new file mode 100644
index 0000000000000000000000000000000000000000..521b5c423c68ba6b3ec7562e39e6137983e77dc6
GIT binary patch
literal 648
zcmb`D+fIcr5Jk~P{{^A6rGkksehIXcUbwXwpnQR!?eWl>cp=fmdD+P%du3+9=$K@X
zTQa|lwn<vB65*gZ%eb!WJ+sTTc;ox<jN_d9$}^X_@MAuXADpMx0}%4j^JY<HI4{$}
z^{e4n7KSjyda|KODQb*LjinRrhJC07%33~Ci3P2V={@0doZk7%e{Vf53Hdi~(GR@=
zubLIyT7;*HAjxq%?S&$}<ZVW{c=r?qFCqWS&$Iu7%dKIhExm8hS@1xXoKX#|5}C^9
tWs;im9mCai(AbXFHA?Z`2>`cL3n8wg;O8!IE8WDiY-4aNzU{MP{{~7(rMCb8

literal 0
HcmV?d00001

diff --git a/tests/unit/libstore/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc
index 6d2054f7d..c2298c6db 100644
--- a/tests/unit/libstore/serve-protocol.cc
+++ b/tests/unit/libstore/serve-protocol.cc
@@ -225,6 +225,83 @@ VERSIONED_CHARACTERIZATION_TEST(
         t;
     }))
 
+VERSIONED_CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    unkeyedValidPathInfo_2_3,
+    "unkeyed-valid-path-info-2.3",
+    2 << 8 | 3,
+    (std::tuple<UnkeyedValidPathInfo, UnkeyedValidPathInfo> {
+        ({
+            UnkeyedValidPathInfo info { Hash::dummy };
+            info.narSize = 34878;
+            info;
+        }),
+        ({
+            UnkeyedValidPathInfo info { Hash::dummy };
+            info.deriver = StorePath {
+                "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
+            };
+            info.references = {
+                StorePath {
+                    "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv",
+                },
+            };
+            info.narSize = 34878;
+            info;
+        }),
+    }))
+
+VERSIONED_CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    unkeyedValidPathInfo_2_4,
+    "unkeyed-valid-path-info-2.4",
+    2 << 8 | 4,
+    (std::tuple<UnkeyedValidPathInfo, UnkeyedValidPathInfo> {
+        ({
+            UnkeyedValidPathInfo info {
+                Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
+            };
+            info.deriver = StorePath {
+                "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
+            };
+            info.references = {
+                StorePath {
+                    "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv",
+                },
+            };
+            info.narSize = 34878;
+            info;
+        }),
+        ({
+            ValidPathInfo info {
+                *LibStoreTest::store,
+                "foo",
+                FixedOutputInfo {
+                    .method = FileIngestionMethod::Recursive,
+                    .hash = hashString(HashAlgorithm::SHA256, "(...)"),
+                    .references = {
+                        .others = {
+                            StorePath {
+                                "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
+                            },
+                        },
+                        .self = true,
+                    },
+                },
+                Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
+            };
+            info.deriver = StorePath {
+                "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
+            };
+            info.narSize = 34878;
+            info.sigs = {
+                "fake-sig-1",
+                "fake-sig-2",
+            },
+            static_cast<UnkeyedValidPathInfo>(std::move(info));
+        }),
+    }))
+
 VERSIONED_CHARACTERIZATION_TEST(
     ServeProtoTest,
     vector,

From 0b80935c22f367b1deecffeddb97c90d7ed985e9 Mon Sep 17 00:00:00 2001
From: Rebecca Turner <rbt@sent.as>
Date: Thu, 7 Dec 2023 10:01:42 -0800
Subject: [PATCH 186/421] Pass positions when evaluating
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This includes position information in more places, making debugging
easier.

Before:

```
$ nix-instantiate --show-trace --eval tests/functional/lang/eval-fail-using-set-as-attr-name.nix
error:
       … while evaluating an attribute name

         at «none»:0: (source not available)

       error: value is a set while a string was expected
```

After:

```
error:
       … while evaluating an attribute name

         at /pwd/lang/eval-fail-using-set-as-attr-name.nix:5:10:

            4| in
            5|   attr.${key}
             |          ^
            6|

       error: value is a set while a string was expected
```
---
 .../rl-next/source-positions-in-errors.md     | 45 +++++++++++++++++++
 src/libexpr/eval-inline.hh                    | 12 ++---
 src/libexpr/eval.cc                           | 18 ++++----
 src/libexpr/nixexpr.hh                        |  1 +
 .../lang/eval-fail-attr-name-type.err.exp     | 20 +++++++++
 .../lang/eval-fail-attr-name-type.nix         |  7 +++
 .../lang/eval-fail-call-primop.err.exp        | 12 +++++
 .../functional/lang/eval-fail-call-primop.nix |  1 +
 .../lang/eval-fail-not-throws.err.exp         | 18 ++++++++
 .../functional/lang/eval-fail-not-throws.nix  |  1 +
 .../eval-fail-using-set-as-attr-name.err.exp  | 11 +++++
 .../lang/eval-fail-using-set-as-attr-name.nix |  5 +++
 12 files changed, 137 insertions(+), 14 deletions(-)
 create mode 100644 doc/manual/rl-next/source-positions-in-errors.md
 create mode 100644 tests/functional/lang/eval-fail-attr-name-type.err.exp
 create mode 100644 tests/functional/lang/eval-fail-attr-name-type.nix
 create mode 100644 tests/functional/lang/eval-fail-call-primop.err.exp
 create mode 100644 tests/functional/lang/eval-fail-call-primop.nix
 create mode 100644 tests/functional/lang/eval-fail-not-throws.err.exp
 create mode 100644 tests/functional/lang/eval-fail-not-throws.nix
 create mode 100644 tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp
 create mode 100644 tests/functional/lang/eval-fail-using-set-as-attr-name.nix

diff --git a/doc/manual/rl-next/source-positions-in-errors.md b/doc/manual/rl-next/source-positions-in-errors.md
new file mode 100644
index 000000000..00f0b27e8
--- /dev/null
+++ b/doc/manual/rl-next/source-positions-in-errors.md
@@ -0,0 +1,45 @@
+synopsis: Source locations are printed more consistently in errors
+issues: #561
+prs: #9555
+description: {
+
+Source location information is now included in error messages more
+consistently. Given this code:
+
+```nix
+let
+  attr = {foo = "bar";};
+  key = {};
+in
+  attr.${key}
+```
+
+Previously, Nix would show this unhelpful message when attempting to evaluate
+it:
+
+```
+error:
+       … while evaluating an attribute name
+
+         at «none»:0: (source not available)
+
+       error: value is a set while a string was expected
+```
+
+Now, the error message displays where the problematic value was found:
+
+```
+error:
+       … while evaluating an attribute name
+
+         at bad.nix:4:11:
+
+            3|   key = {};
+            4| in attr.${key}
+             |           ^
+            5|
+
+       error: value is a set while a string was expected
+```
+
+}
diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index a988fa40c..c37b1d62b 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -103,8 +103,10 @@ void EvalState::forceValue(Value & v, Callable getPos)
             throw;
         }
     }
-    else if (v.isApp())
-        callFunction(*v.app.left, *v.app.right, v, noPos);
+    else if (v.isApp()) {
+        PosIdx pos = getPos();
+        callFunction(*v.app.left, *v.app.right, v, pos);
+    }
     else if (v.isBlackhole())
         error("infinite recursion encountered").atPos(getPos()).template debugThrow<EvalError>();
 }
@@ -121,9 +123,9 @@ template <typename Callable>
 [[gnu::always_inline]]
 inline void EvalState::forceAttrs(Value & v, Callable getPos, std::string_view errorCtx)
 {
-    forceValue(v, noPos);
+    PosIdx pos = getPos();
+    forceValue(v, pos);
     if (v.type() != nAttrs) {
-        PosIdx pos = getPos();
         error("value is %1% while a set was expected", showType(v)).withTrace(pos, errorCtx).debugThrow<TypeError>();
     }
 }
@@ -132,7 +134,7 @@ inline void EvalState::forceAttrs(Value & v, Callable getPos, std::string_view e
 [[gnu::always_inline]]
 inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view errorCtx)
 {
-    forceValue(v, noPos);
+    forceValue(v, pos);
     if (!v.isList()) {
         error("value is %1% while a list was expected", showType(v)).withTrace(pos, errorCtx).debugThrow<TypeError>();
     }
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 7e68e6f9b..8a6e07fb0 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -344,7 +344,7 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env)
     } else {
         Value nameValue;
         name.expr->eval(state, env, nameValue);
-        state.forceStringNoCtx(nameValue, noPos, "while evaluating an attribute name");
+        state.forceStringNoCtx(nameValue, name.expr->getPos(), "while evaluating an attribute name");
         return state.symbols.create(nameValue.string_view());
     }
 }
@@ -1514,7 +1514,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v)
     e->eval(state, env, vTmp);
 
     for (auto & i : attrPath) {
-        state.forceValue(*vAttrs, noPos);
+        state.forceValue(*vAttrs, getPos());
         Bindings::iterator j;
         auto name = getName(i, state, env);
         if (vAttrs->type() != nAttrs ||
@@ -1683,7 +1683,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 if (countCalls) primOpCalls[name]++;
 
                 try {
-                    vCur.primOp->fun(*this, noPos, args, vCur);
+                    vCur.primOp->fun(*this, vCur.determinePos(noPos), args, vCur);
                 } catch (Error & e) {
                     addErrorTrace(e, pos, "while calling the '%1%' builtin", name);
                     throw;
@@ -1731,7 +1731,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                     // 1. Unify this and above code. Heavily redundant.
                     // 2. Create a fake env (arg1, arg2, etc.) and a fake expr (arg1: arg2: etc: builtins.name arg1 arg2 etc)
                     //    so the debugger allows to inspect the wrong parameters passed to the builtin.
-                    primOp->primOp->fun(*this, noPos, vArgs, vCur);
+                    primOp->primOp->fun(*this, vCur.determinePos(noPos), vArgs, vCur);
                 } catch (Error & e) {
                     addErrorTrace(e, pos, "while calling the '%1%' builtin", name);
                     throw;
@@ -1839,7 +1839,7 @@ https://nixos.org/manual/nix/stable/language/constructs.html#functions.)", symbo
         }
     }
 
-    callFunction(fun, allocValue()->mkAttrs(attrs), res, noPos);
+    callFunction(fun, allocValue()->mkAttrs(attrs), res, pos);
 }
 
 
@@ -1875,7 +1875,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
 
 void ExprOpNot::eval(EvalState & state, Env & env, Value & v)
 {
-    v.mkBool(!state.evalBool(env, e, noPos, "in the argument of the not operator")); // XXX: FIXME: !
+    v.mkBool(!state.evalBool(env, e, getPos(), "in the argument of the not operator")); // XXX: FIXME: !
 }
 
 
@@ -2316,7 +2316,7 @@ BackedStringView EvalState::coerceToString(
             std::string result;
             for (auto [n, v2] : enumerate(v.listItems())) {
                 try {
-                    result += *coerceToString(noPos, *v2, context,
+                    result += *coerceToString(pos, *v2, context,
                             "while evaluating one element of the list",
                             coerceMore, copyToStore, canonicalizePath);
                 } catch (Error & e) {
@@ -2463,8 +2463,8 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value &
 
 bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx)
 {
-    forceValue(v1, noPos);
-    forceValue(v2, noPos);
+    forceValue(v1, pos);
+    forceValue(v2, pos);
 
     /* !!! Hack to support some old broken code that relies on pointer
        equality tests between sets.  (Specifically, builderDefs calls
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index 10099d49e..020286815 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -405,6 +405,7 @@ struct ExprOpNot : Expr
 {
     Expr * e;
     ExprOpNot(Expr * e) : e(e) { };
+    PosIdx getPos() const override { return e->getPos(); }
     COMMON_METHODS
 };
 
diff --git a/tests/functional/lang/eval-fail-attr-name-type.err.exp b/tests/functional/lang/eval-fail-attr-name-type.err.exp
new file mode 100644
index 000000000..5f9a073dd
--- /dev/null
+++ b/tests/functional/lang/eval-fail-attr-name-type.err.exp
@@ -0,0 +1,20 @@
+error:
+       … while evaluating the attribute 'puppy."${key}"'
+
+         at /pwd/lang/eval-fail-attr-name-type.nix:3:5:
+
+            2|   attrs = {
+            3|     puppy.doggy = {};
+             |     ^
+            4|   };
+
+       … while evaluating an attribute name
+
+         at /pwd/lang/eval-fail-attr-name-type.nix:7:17:
+
+            6| in
+            7|   attrs.puppy.${key}
+             |                 ^
+            8|
+
+       error: value is an integer while a string was expected
diff --git a/tests/functional/lang/eval-fail-attr-name-type.nix b/tests/functional/lang/eval-fail-attr-name-type.nix
new file mode 100644
index 000000000..a0e76004a
--- /dev/null
+++ b/tests/functional/lang/eval-fail-attr-name-type.nix
@@ -0,0 +1,7 @@
+let
+  attrs = {
+    puppy.doggy = {};
+  };
+  key = 1;
+in
+  attrs.puppy.${key}
diff --git a/tests/functional/lang/eval-fail-call-primop.err.exp b/tests/functional/lang/eval-fail-call-primop.err.exp
new file mode 100644
index 000000000..19b407c47
--- /dev/null
+++ b/tests/functional/lang/eval-fail-call-primop.err.exp
@@ -0,0 +1,12 @@
+error:
+       … while calling the 'length' builtin
+
+         at /pwd/lang/eval-fail-call-primop.nix:1:1:
+
+            1| builtins.length 1
+             | ^
+            2|
+
+       … while evaluating the first argument passed to builtins.length
+
+       error: value is an integer while a list was expected
diff --git a/tests/functional/lang/eval-fail-call-primop.nix b/tests/functional/lang/eval-fail-call-primop.nix
new file mode 100644
index 000000000..972eb72c7
--- /dev/null
+++ b/tests/functional/lang/eval-fail-call-primop.nix
@@ -0,0 +1 @@
+builtins.length 1
diff --git a/tests/functional/lang/eval-fail-not-throws.err.exp b/tests/functional/lang/eval-fail-not-throws.err.exp
new file mode 100644
index 000000000..b290afb0a
--- /dev/null
+++ b/tests/functional/lang/eval-fail-not-throws.err.exp
@@ -0,0 +1,18 @@
+error:
+       … in the argument of the not operator
+
+         at /pwd/lang/eval-fail-not-throws.nix:1:4:
+
+            1| ! (throw "uh oh!")
+             |    ^
+            2|
+
+       … while calling the 'throw' builtin
+
+         at /pwd/lang/eval-fail-not-throws.nix:1:4:
+
+            1| ! (throw "uh oh!")
+             |    ^
+            2|
+
+       error: uh oh!
diff --git a/tests/functional/lang/eval-fail-not-throws.nix b/tests/functional/lang/eval-fail-not-throws.nix
new file mode 100644
index 000000000..a74ce4ebe
--- /dev/null
+++ b/tests/functional/lang/eval-fail-not-throws.nix
@@ -0,0 +1 @@
+! (throw "uh oh!")
diff --git a/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp b/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp
new file mode 100644
index 000000000..811d01b03
--- /dev/null
+++ b/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp
@@ -0,0 +1,11 @@
+error:
+       … while evaluating an attribute name
+
+         at /pwd/lang/eval-fail-using-set-as-attr-name.nix:5:10:
+
+            4| in
+            5|   attr.${key}
+             |          ^
+            6|
+
+       error: value is a set while a string was expected
diff --git a/tests/functional/lang/eval-fail-using-set-as-attr-name.nix b/tests/functional/lang/eval-fail-using-set-as-attr-name.nix
new file mode 100644
index 000000000..48e071a41
--- /dev/null
+++ b/tests/functional/lang/eval-fail-using-set-as-attr-name.nix
@@ -0,0 +1,5 @@
+let
+  attr = {foo = "bar";};
+  key = {};
+in
+  attr.${key}

From 96dd757b0c0f3d6702f8e38467a8bf467b43154e Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 8 Dec 2023 00:44:55 -0500
Subject: [PATCH 187/421] Give `Derivation::tryResolve` an `evalStore` argument

This is needed for building CA deriations with a src store / dest store
split. In particular it is needed for Hydra.

https://github.com/NixOS/hydra/issues/838 currently puts realizations,
and thus build outputs, in the local store, but it should not.
---
 src/libstore/build/derivation-goal.cc | 2 +-
 src/libstore/derivations.cc           | 4 ++--
 src/libstore/derivations.hh           | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 81eef7c47..d4da374ba 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -558,7 +558,7 @@ void DerivationGoal::inputsRealised()
                  inputDrvOutputs statefully, sometimes it gets out of sync with
                  the real source of truth (store). So we query the store
                  directly if there's a problem. */
-              attempt = fullDrv.tryResolve(worker.store);
+              attempt = fullDrv.tryResolve(worker.store, &worker.evalStore);
             }
             assert(attempt);
             Derivation drvResolved { std::move(*attempt) };
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 664ab7556..c35150b57 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -1002,13 +1002,13 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
 
 }
 
-std::optional<BasicDerivation> Derivation::tryResolve(Store & store) const
+std::optional<BasicDerivation> Derivation::tryResolve(Store & store, Store * evalStore) const
 {
     std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
 
     std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accum;
     accum = [&](auto & inputDrv, auto & node) {
-        for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv)) {
+        for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv, evalStore)) {
             if (outputPath) {
                 inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath);
                 if (auto p = get(node.childMap, outputName))
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 290abedcf..2a326b578 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -342,7 +342,7 @@ struct Derivation : BasicDerivation
      * 2. Input placeholders are replaced with realized input store
      *    paths.
      */
-    std::optional<BasicDerivation> tryResolve(Store & store) const;
+    std::optional<BasicDerivation> tryResolve(Store & store, Store * evalStore = nullptr) const;
 
     /**
      * Like the above, but instead of querying the Nix database for

From f0ac2a35d5e9dfb3a53e6cc810e871fe119cbf4b Mon Sep 17 00:00:00 2001
From: Rebecca Turner <rbt@sent.as>
Date: Fri, 8 Dec 2023 11:36:57 -0500
Subject: [PATCH 188/421] Print the value in `error: cannot coerce` messages
 (#9553)

* Print the value in `error: cannot coerce` messages

This extends the `error: cannot coerce a TYPE to a string` message
to print the value that could not be coerced. This helps with debugging
by making it easier to track down where the value is being produced
from, especially in errors with deep or unhelpful stack traces.

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 .../rl-next/print-value-in-coercion-error.md  | 50 +++++++++++++++++++
 .../src/language/string-interpolation.md      |  2 +-
 src/libexpr/eval.cc                           | 10 ++--
 ...al-fail-bad-string-interpolation-1.err.exp |  2 +-
 ...al-fail-bad-string-interpolation-3.err.exp |  2 +-
 tests/unit/libexpr/error_traces.cc            | 28 +++++------
 6 files changed, 73 insertions(+), 21 deletions(-)
 create mode 100644 doc/manual/rl-next/print-value-in-coercion-error.md

diff --git a/doc/manual/rl-next/print-value-in-coercion-error.md b/doc/manual/rl-next/print-value-in-coercion-error.md
new file mode 100644
index 000000000..504ea67b9
--- /dev/null
+++ b/doc/manual/rl-next/print-value-in-coercion-error.md
@@ -0,0 +1,50 @@
+synopsis: Coercion errors include the failing value
+issues: #561
+prs: #9553
+description: {
+
+The `error: cannot coerce a <TYPE> to a string` message now includes the value which caused the error.
+
+Previously, a failed string coercion produced a confusing error message if the trace didn't show where the offending value was defined:
+
+```bash
+$ nix-instantiate --eval --expr '
+let x = { a = 1; }; in
+
+"${x}"
+'
+error:
+       … while evaluating a path segment
+
+         at «string»:4:2:
+
+            3|
+            4| "${x}"
+             |  ^
+            5|
+
+       error: cannot coerce a set to a string
+```
+
+Now, the error message includes the value itself:
+
+```bash
+$ nix-instantiate --eval --expr '
+let x = { a = 1; }; in
+
+"${x}"
+'
+error:
+       … while evaluating a path segment
+
+         at «string»:4:2:
+
+            3|
+            4| "${x}"
+             |  ^
+            5|
+
+       error: cannot coerce a set to a string: { a = 1; }
+```
+
+}
diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md
index e999b287b..6e28d2664 100644
--- a/doc/manual/src/language/string-interpolation.md
+++ b/doc/manual/src/language/string-interpolation.md
@@ -189,7 +189,7 @@ If neither is present, an error is thrown.
 > "${a}"
 > ```
 >
->     error: cannot coerce a set to a string
+>     error: cannot coerce a set to a string: { }
 >
 >            at «string»:4:2:
 >
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 7e68e6f9b..b52274b64 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -26,9 +26,9 @@
 #include <unistd.h>
 #include <sys/time.h>
 #include <sys/resource.h>
-#include <iostream>
 #include <fstream>
 #include <functional>
+#include <strstream>
 
 #include <sys/resource.h>
 #include <nlohmann/json.hpp>
@@ -2286,7 +2286,7 @@ BackedStringView EvalState::coerceToString(
             return std::move(*maybeString);
         auto i = v.attrs->find(sOutPath);
         if (i == v.attrs->end()) {
-            error("cannot coerce %1% to a string", showType(v))
+            error("cannot coerce %1% to a string: %2%", showType(v), printValue(*this, v))
                 .withTrace(pos, errorCtx)
                 .debugThrow<TypeError>();
         }
@@ -2332,7 +2332,7 @@ BackedStringView EvalState::coerceToString(
         }
     }
 
-    error("cannot coerce %1% to a string", showType(v))
+    error("cannot coerce %1% to a string: %2%", showType(v), printValue(*this, v))
         .withTrace(pos, errorCtx)
         .debugThrow<TypeError>();
 }
@@ -2691,8 +2691,10 @@ void EvalState::printStatistics()
 
 std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const
 {
+    std::strstream printed;
+    print(printed);
     throw TypeError({
-        .msg = hintfmt("cannot coerce %1% to a string", showType())
+        .msg = hintfmt("cannot coerce %1% to a string: %2%", showType(), printed.str())
     });
 }
 
diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp
index eb73e9a52..e54ecc6d1 100644
--- a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp
+++ b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp
@@ -7,4 +7,4 @@ error:
              |  ^
             2|
 
-       error: cannot coerce a function to a string
+       error: cannot coerce a function to a string: <LAMBDA>
diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp
index ac14f329b..6f0a96f78 100644
--- a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp
+++ b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp
@@ -7,4 +7,4 @@ error:
              |   ^
             2|
 
-       error: cannot coerce a function to a string
+       error: cannot coerce a function to a string: <LAMBDA>
diff --git a/tests/unit/libexpr/error_traces.cc b/tests/unit/libexpr/error_traces.cc
index 81498f65a..c2403bee9 100644
--- a/tests/unit/libexpr/error_traces.cc
+++ b/tests/unit/libexpr/error_traces.cc
@@ -295,7 +295,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, toPath) {
         ASSERT_TRACE2("toPath []",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a list"),
+                      hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"),
                       hintfmt("while evaluating the first argument passed to builtins.toPath"));
 
         ASSERT_TRACE2("toPath \"foo\"",
@@ -309,7 +309,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, storePath) {
         ASSERT_TRACE2("storePath true",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a Boolean"),
+                      hintfmt("cannot coerce %s to a string: %s", "a Boolean", "true"),
                       hintfmt("while evaluating the first argument passed to 'builtins.storePath'"));
 
     }
@@ -318,7 +318,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, pathExists) {
         ASSERT_TRACE2("pathExists []",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a list"),
+                      hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"),
                       hintfmt("while realising the context of a path"));
 
         ASSERT_TRACE2("pathExists \"zorglub\"",
@@ -332,7 +332,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, baseNameOf) {
         ASSERT_TRACE2("baseNameOf []",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a list"),
+                      hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"),
                       hintfmt("while evaluating the first argument passed to builtins.baseNameOf"));
 
     }
@@ -377,7 +377,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, filterSource) {
         ASSERT_TRACE2("filterSource [] []",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a list"),
+                      hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"),
                       hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"));
 
         ASSERT_TRACE2("filterSource [] \"foo\"",
@@ -1038,7 +1038,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, toString) {
         ASSERT_TRACE2("toString { a = 1; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a set"),
+                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ a = 1; }"),
                       hintfmt("while evaluating the first argument passed to builtins.toString"));
 
     }
@@ -1057,7 +1057,7 @@ namespace nix {
 
         ASSERT_TRACE2("substring 0 3 {}",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a set"),
+                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
                       hintfmt("while evaluating the third argument (the string) passed to builtins.substring"));
 
         ASSERT_TRACE1("substring (-3) 3 \"sometext\"",
@@ -1070,7 +1070,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, stringLength) {
         ASSERT_TRACE2("stringLength {} # TODO: context is missing ???",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a set"),
+                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
                       hintfmt("while evaluating the argument passed to builtins.stringLength"));
 
     }
@@ -1143,7 +1143,7 @@ namespace nix {
 
         ASSERT_TRACE2("concatStringsSep \"foo\" [ 1 2 {} ] # TODO: coerce to string is buggy",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "an integer"),
+                      hintfmt("cannot coerce %s to a string: %s", "an integer", "1"),
                       hintfmt("while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep"));
 
     }
@@ -1229,12 +1229,12 @@ namespace nix {
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = {}; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a set"),
+                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
                       hintfmt("while evaluating the attribute 'system' of derivation 'foo'"));
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = {}; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a set"),
+                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
                       hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'"));
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"drv\"; }",
@@ -1279,17 +1279,17 @@ namespace nix {
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ {} ]; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a set"),
+                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
                       hintfmt("while evaluating an element of the argument list"));
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ \"a\" {} ]; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a set"),
+                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
                       hintfmt("while evaluating an element of the argument list"));
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; FOO = {}; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string", "a set"),
+                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
                       hintfmt("while evaluating the attribute 'FOO' of derivation 'foo'"));
 
     }

From f9ee1bedcf98334d8bc015c2e04e30fbba958a3e Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 8 Dec 2023 13:18:52 -0500
Subject: [PATCH 189/421] Avoid `std::strstream`, fix the clang build

According https://en.cppreference.com/w/cpp/io/strstream, it has been
deprecated since C++98! The Clang + Linux build systems to not have it
at all, or at least be hiding it.

We can just use `std::stringstream` instead, I think.
---
 src/libexpr/eval.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index b52274b64..5d627224f 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -28,7 +28,7 @@
 #include <sys/resource.h>
 #include <fstream>
 #include <functional>
-#include <strstream>
+#include <sstream>
 
 #include <sys/resource.h>
 #include <nlohmann/json.hpp>
@@ -2691,7 +2691,7 @@ void EvalState::printStatistics()
 
 std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const
 {
-    std::strstream printed;
+    std::stringstream printed;
     print(printed);
     throw TypeError({
         .msg = hintfmt("cannot coerce %1% to a string: %2%", showType(), printed.str())

From 005eaa1bd6c6090d5a55a062f429e6464345c6df Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 1 Dec 2023 16:40:54 +0100
Subject: [PATCH 190/421] doc/prerequisites-source: Add
 bdwgc-traceable-allocator patch

---
 doc/manual/src/installation/prerequisites-source.md | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md
index d4babf1ea..907d7f63f 100644
--- a/doc/manual/src/installation/prerequisites-source.md
+++ b/doc/manual/src/installation/prerequisites-source.md
@@ -32,11 +32,15 @@
     your distribution does not provide it, please install it from
     <http://www.sqlite.org/>.
 
-  - The [Boehm garbage collector](http://www.hboehm.info/gc/) to reduce
-    the evaluator’s memory consumption (optional). To enable it, install
+  - The [Boehm garbage collector (`bdw-gc`)](http://www.hboehm.info/gc/) to reduce
+    the evaluator’s memory consumption (optional).
+
+    To enable it, install
     `pkgconfig` and the Boehm garbage collector, and pass the flag
     `--enable-gc` to `configure`.
 
+    For `bdw-gc` <= 8.2.4 Nix needs a [small patch](https://github.com/NixOS/nix/blob/ac4d2e7b857acdfeac35ac8a592bdecee2d29838/boehmgc-traceable_allocator-public.diff) to be applied.
+
   - The `boost` library of version 1.66.0 or higher. It can be obtained
     from the official web site <https://www.boost.org/>.
 

From ce4ca574d24abe233b717babc679e4c9228ba94b Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 6 Nov 2023 09:04:50 -0500
Subject: [PATCH 191/421] Clarify `SourceAccessor` methods should never
 implicitly follow symlinks

The code has already been fixed (yay!) so what is left of this commit is
just updating the API docs.

Co-authored-by: Cole Helbling <cole.e.helbling@outlook.com>
---
 src/libutil/source-accessor.hh | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh
index 3ca12d624..4f4ff09c1 100644
--- a/src/libutil/source-accessor.hh
+++ b/src/libutil/source-accessor.hh
@@ -26,6 +26,13 @@ struct SourceAccessor
 
     /**
      * Return the contents of a file as a string.
+     *
+     * @note Unlike Unix, this method should *not* follow symlinks. Nix
+     * by default wants to manipulate symlinks explicitly, and not
+     * implictly follow them, as they are frequently untrusted user data
+     * and thus may point to arbitrary locations. Acting on the targets
+     * targets of symlinks should only occasionally be done, and only
+     * with care.
      */
     virtual std::string readFile(const CanonPath & path);
 
@@ -34,7 +41,10 @@ struct SourceAccessor
      * called with the size of the file before any data is written to
      * the sink.
      *
-     * Note: subclasses of `SourceAccessor` need to implement at least
+     * @note Like the other `readFile`, this method should *not* follow
+     * symlinks.
+     *
+     * @note subclasses of `SourceAccessor` need to implement at least
      * one of the `readFile()` variants.
      */
     virtual void readFile(
@@ -87,6 +97,9 @@ struct SourceAccessor
 
     typedef std::map<std::string, DirEntry> DirEntries;
 
+    /**
+     * @note Like `readFile`, this method should *not* follow symlinks.
+     */
     virtual DirEntries readDirectory(const CanonPath & path) = 0;
 
     virtual std::string readLink(const CanonPath & path) = 0;

From 9b7b7a7561b24d48452627709e6872d9c610428b Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sat, 9 Dec 2023 02:13:32 +0100
Subject: [PATCH 192/421] Revert "Print the value in `error: cannot coerce`
 messages (#9553)"

This reverts commit f0ac2a35d5e9dfb3a53e6cc810e871fe119cbf4b.

The request from the sibling PR, which also applies here, was not addressed.
https://github.com/NixOS/nix/pull/9554#issuecomment-1845095735
---
 .../rl-next/print-value-in-coercion-error.md  | 50 -------------------
 .../src/language/string-interpolation.md      |  2 +-
 src/libexpr/eval.cc                           | 10 ++--
 ...al-fail-bad-string-interpolation-1.err.exp |  2 +-
 ...al-fail-bad-string-interpolation-3.err.exp |  2 +-
 tests/unit/libexpr/error_traces.cc            | 28 +++++------
 6 files changed, 21 insertions(+), 73 deletions(-)
 delete mode 100644 doc/manual/rl-next/print-value-in-coercion-error.md

diff --git a/doc/manual/rl-next/print-value-in-coercion-error.md b/doc/manual/rl-next/print-value-in-coercion-error.md
deleted file mode 100644
index 504ea67b9..000000000
--- a/doc/manual/rl-next/print-value-in-coercion-error.md
+++ /dev/null
@@ -1,50 +0,0 @@
-synopsis: Coercion errors include the failing value
-issues: #561
-prs: #9553
-description: {
-
-The `error: cannot coerce a <TYPE> to a string` message now includes the value which caused the error.
-
-Previously, a failed string coercion produced a confusing error message if the trace didn't show where the offending value was defined:
-
-```bash
-$ nix-instantiate --eval --expr '
-let x = { a = 1; }; in
-
-"${x}"
-'
-error:
-       … while evaluating a path segment
-
-         at «string»:4:2:
-
-            3|
-            4| "${x}"
-             |  ^
-            5|
-
-       error: cannot coerce a set to a string
-```
-
-Now, the error message includes the value itself:
-
-```bash
-$ nix-instantiate --eval --expr '
-let x = { a = 1; }; in
-
-"${x}"
-'
-error:
-       … while evaluating a path segment
-
-         at «string»:4:2:
-
-            3|
-            4| "${x}"
-             |  ^
-            5|
-
-       error: cannot coerce a set to a string: { a = 1; }
-```
-
-}
diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md
index 6e28d2664..e999b287b 100644
--- a/doc/manual/src/language/string-interpolation.md
+++ b/doc/manual/src/language/string-interpolation.md
@@ -189,7 +189,7 @@ If neither is present, an error is thrown.
 > "${a}"
 > ```
 >
->     error: cannot coerce a set to a string: { }
+>     error: cannot coerce a set to a string
 >
 >            at «string»:4:2:
 >
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index c04e2d53d..841c223cd 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -27,9 +27,9 @@
 #include <unistd.h>
 #include <sys/time.h>
 #include <sys/resource.h>
+#include <iostream>
 #include <fstream>
 #include <functional>
-#include <sstream>
 
 #include <sys/resource.h>
 #include <nlohmann/json.hpp>
@@ -2230,7 +2230,7 @@ BackedStringView EvalState::coerceToString(
             return std::move(*maybeString);
         auto i = v.attrs->find(sOutPath);
         if (i == v.attrs->end()) {
-            error("cannot coerce %1% to a string: %2%", showType(v), printValue(*this, v))
+            error("cannot coerce %1% to a string", showType(v))
                 .withTrace(pos, errorCtx)
                 .debugThrow<TypeError>();
         }
@@ -2276,7 +2276,7 @@ BackedStringView EvalState::coerceToString(
         }
     }
 
-    error("cannot coerce %1% to a string: %2%", showType(v), printValue(*this, v))
+    error("cannot coerce %1% to a string", showType(v))
         .withTrace(pos, errorCtx)
         .debugThrow<TypeError>();
 }
@@ -2635,10 +2635,8 @@ void EvalState::printStatistics()
 
 std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const
 {
-    std::stringstream printed;
-    print(printed);
     throw TypeError({
-        .msg = hintfmt("cannot coerce %1% to a string: %2%", showType(), printed.str())
+        .msg = hintfmt("cannot coerce %1% to a string", showType())
     });
 }
 
diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp
index e54ecc6d1..eb73e9a52 100644
--- a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp
+++ b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp
@@ -7,4 +7,4 @@ error:
              |  ^
             2|
 
-       error: cannot coerce a function to a string: <LAMBDA>
+       error: cannot coerce a function to a string
diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp
index 6f0a96f78..ac14f329b 100644
--- a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp
+++ b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp
@@ -7,4 +7,4 @@ error:
              |   ^
             2|
 
-       error: cannot coerce a function to a string: <LAMBDA>
+       error: cannot coerce a function to a string
diff --git a/tests/unit/libexpr/error_traces.cc b/tests/unit/libexpr/error_traces.cc
index c2403bee9..81498f65a 100644
--- a/tests/unit/libexpr/error_traces.cc
+++ b/tests/unit/libexpr/error_traces.cc
@@ -295,7 +295,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, toPath) {
         ASSERT_TRACE2("toPath []",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"),
+                      hintfmt("cannot coerce %s to a string", "a list"),
                       hintfmt("while evaluating the first argument passed to builtins.toPath"));
 
         ASSERT_TRACE2("toPath \"foo\"",
@@ -309,7 +309,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, storePath) {
         ASSERT_TRACE2("storePath true",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a Boolean", "true"),
+                      hintfmt("cannot coerce %s to a string", "a Boolean"),
                       hintfmt("while evaluating the first argument passed to 'builtins.storePath'"));
 
     }
@@ -318,7 +318,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, pathExists) {
         ASSERT_TRACE2("pathExists []",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"),
+                      hintfmt("cannot coerce %s to a string", "a list"),
                       hintfmt("while realising the context of a path"));
 
         ASSERT_TRACE2("pathExists \"zorglub\"",
@@ -332,7 +332,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, baseNameOf) {
         ASSERT_TRACE2("baseNameOf []",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"),
+                      hintfmt("cannot coerce %s to a string", "a list"),
                       hintfmt("while evaluating the first argument passed to builtins.baseNameOf"));
 
     }
@@ -377,7 +377,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, filterSource) {
         ASSERT_TRACE2("filterSource [] []",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"),
+                      hintfmt("cannot coerce %s to a string", "a list"),
                       hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"));
 
         ASSERT_TRACE2("filterSource [] \"foo\"",
@@ -1038,7 +1038,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, toString) {
         ASSERT_TRACE2("toString { a = 1; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ a = 1; }"),
+                      hintfmt("cannot coerce %s to a string", "a set"),
                       hintfmt("while evaluating the first argument passed to builtins.toString"));
 
     }
@@ -1057,7 +1057,7 @@ namespace nix {
 
         ASSERT_TRACE2("substring 0 3 {}",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
+                      hintfmt("cannot coerce %s to a string", "a set"),
                       hintfmt("while evaluating the third argument (the string) passed to builtins.substring"));
 
         ASSERT_TRACE1("substring (-3) 3 \"sometext\"",
@@ -1070,7 +1070,7 @@ namespace nix {
     TEST_F(ErrorTraceTest, stringLength) {
         ASSERT_TRACE2("stringLength {} # TODO: context is missing ???",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
+                      hintfmt("cannot coerce %s to a string", "a set"),
                       hintfmt("while evaluating the argument passed to builtins.stringLength"));
 
     }
@@ -1143,7 +1143,7 @@ namespace nix {
 
         ASSERT_TRACE2("concatStringsSep \"foo\" [ 1 2 {} ] # TODO: coerce to string is buggy",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "an integer", "1"),
+                      hintfmt("cannot coerce %s to a string", "an integer"),
                       hintfmt("while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep"));
 
     }
@@ -1229,12 +1229,12 @@ namespace nix {
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = {}; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
+                      hintfmt("cannot coerce %s to a string", "a set"),
                       hintfmt("while evaluating the attribute 'system' of derivation 'foo'"));
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = {}; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
+                      hintfmt("cannot coerce %s to a string", "a set"),
                       hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'"));
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"drv\"; }",
@@ -1279,17 +1279,17 @@ namespace nix {
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ {} ]; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
+                      hintfmt("cannot coerce %s to a string", "a set"),
                       hintfmt("while evaluating an element of the argument list"));
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ \"a\" {} ]; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
+                      hintfmt("cannot coerce %s to a string", "a set"),
                       hintfmt("while evaluating an element of the argument list"));
 
         ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; FOO = {}; }",
                       TypeError,
-                      hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"),
+                      hintfmt("cannot coerce %s to a string", "a set"),
                       hintfmt("while evaluating the attribute 'FOO' of derivation 'foo'"));
 
     }

From b9980b377ede0aca542b2baeeef9e4538dec20db Mon Sep 17 00:00:00 2001
From: Robert Hensing <roberth@users.noreply.github.com>
Date: Sat, 9 Dec 2023 02:36:33 +0100
Subject: [PATCH 193/421] Update rl-next/source-positions-in-errors for Nix
 2.19+

---
 doc/manual/rl-next/source-positions-in-errors.md | 2 --
 1 file changed, 2 deletions(-)

diff --git a/doc/manual/rl-next/source-positions-in-errors.md b/doc/manual/rl-next/source-positions-in-errors.md
index 00f0b27e8..15df884ea 100644
--- a/doc/manual/rl-next/source-positions-in-errors.md
+++ b/doc/manual/rl-next/source-positions-in-errors.md
@@ -21,8 +21,6 @@ it:
 error:
        … while evaluating an attribute name
 
-         at «none»:0: (source not available)
-
        error: value is a set while a string was expected
 ```
 

From 6e8d5983143ae576e3f4b1d2954a5267f2943a49 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sat, 9 Dec 2023 02:17:36 +0100
Subject: [PATCH 194/421] tests/lang/eval-fail-bad-string-interpolation-4: init

---
 .../lang/eval-fail-bad-string-interpolation-4.err.exp | 11 +++++++++++
 .../lang/eval-fail-bad-string-interpolation-4.nix     |  9 +++++++++
 2 files changed, 20 insertions(+)
 create mode 100644 tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp
 create mode 100644 tests/functional/lang/eval-fail-bad-string-interpolation-4.nix

diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp
new file mode 100644
index 000000000..07843a480
--- /dev/null
+++ b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp
@@ -0,0 +1,11 @@
+error:
+       … while evaluating a path segment
+
+         at /pwd/lang/eval-fail-bad-string-interpolation-4.nix:9:3:
+
+            8| # The error message should not be too long.
+            9| ''${pkgs}''
+             |   ^
+           10|
+
+       error: cannot coerce a set to a string
diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-4.nix b/tests/functional/lang/eval-fail-bad-string-interpolation-4.nix
new file mode 100644
index 000000000..457b5f06a
--- /dev/null
+++ b/tests/functional/lang/eval-fail-bad-string-interpolation-4.nix
@@ -0,0 +1,9 @@
+let
+  # Basically a "billion laughs" attack, but toned down to simulated `pkgs`.
+  ha = x: y: { a = x y; b = x y; c = x y; d = x y; e = x y; f = x y; g = x y; h = x y; j = x y; };
+  has = ha (ha (ha (ha (x: x)))) "ha";
+  # A large structure that has already been evaluated.
+  pkgs = builtins.deepSeq has has;
+in
+# The error message should not be too long.
+''${pkgs}''

From 5417990e313272a5f1129ac39228b111e8dac857 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 8 Dec 2023 14:32:22 -0500
Subject: [PATCH 195/421] Create `ServeProto::BuildOptions` and a serializer
 for it

More tests, and more serializers for Hydra reuse.
---
 src/libstore/legacy-ssh-store.cc              |  22 +++-----
 src/libstore/serve-protocol.cc                |  36 +++++++++++++
 src/libstore/serve-protocol.hh                |  25 +++++++++
 src/nix-store/nix-store.cc                    |  34 ++++++++-----
 .../data/serve-protocol/build-options-2.1.bin | Bin 0 -> 16 bytes
 .../data/serve-protocol/build-options-2.2.bin | Bin 0 -> 24 bytes
 .../data/serve-protocol/build-options-2.3.bin | Bin 0 -> 40 bytes
 .../data/serve-protocol/build-options-2.7.bin | Bin 0 -> 48 bytes
 tests/unit/libstore/serve-protocol.cc         |  48 ++++++++++++++++++
 9 files changed, 137 insertions(+), 28 deletions(-)
 create mode 100644 tests/unit/libstore/data/serve-protocol/build-options-2.1.bin
 create mode 100644 tests/unit/libstore/data/serve-protocol/build-options-2.2.bin
 create mode 100644 tests/unit/libstore/data/serve-protocol/build-options-2.3.bin
 create mode 100644 tests/unit/libstore/data/serve-protocol/build-options-2.7.bin

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 277445ee6..8ef2daa7b 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -275,20 +275,14 @@ private:
 
     void putBuildSettings(Connection & conn)
     {
-        conn.to
-            << settings.maxSilentTime
-            << settings.buildTimeout;
-        if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 2)
-            conn.to
-                << settings.maxLogSize;
-        if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3)
-            conn.to
-                << 0 // buildRepeat hasn't worked for ages anyway
-                << 0;
-
-        if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 7) {
-            conn.to << ((int) settings.keepFailed);
-        }
+        ServeProto::write(*this, conn, ServeProto::BuildOptions {
+            .maxSilentTime = settings.maxSilentTime,
+            .buildTimeout = settings.buildTimeout,
+            .maxLogSize = settings.maxLogSize,
+            .nrRepeats = 0, // buildRepeat hasn't worked for ages anyway
+            .enforceDeterminism = 0,
+            .keepFailed = settings.keepFailed,
+        });
     }
 
 public:
diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc
index c37b3095c..08bfad9e4 100644
--- a/src/libstore/serve-protocol.cc
+++ b/src/libstore/serve-protocol.cc
@@ -98,4 +98,40 @@ void ServeProto::Serialise<UnkeyedValidPathInfo>::write(const StoreDirConfig & s
             << info.sigs;
 }
 
+
+ServeProto::BuildOptions ServeProto::Serialise<ServeProto::BuildOptions>::read(const StoreDirConfig & store, ReadConn conn)
+{
+    BuildOptions options;
+    options.maxSilentTime = readInt(conn.from);
+    options.buildTimeout = readInt(conn.from);
+    if (GET_PROTOCOL_MINOR(conn.version) >= 2)
+        options.maxLogSize = readNum<unsigned long>(conn.from);
+    if (GET_PROTOCOL_MINOR(conn.version) >= 3) {
+        options.nrRepeats = readInt(conn.from);
+        options.enforceDeterminism = readInt(conn.from);
+    }
+    if (GET_PROTOCOL_MINOR(conn.version) >= 7) {
+        options.keepFailed = (bool) readInt(conn.from);
+    }
+    return options;
+}
+
+void ServeProto::Serialise<ServeProto::BuildOptions>::write(const StoreDirConfig & store, WriteConn conn, const ServeProto::BuildOptions & options)
+{
+    conn.to
+        << options.maxSilentTime
+        << options.buildTimeout;
+    if (GET_PROTOCOL_MINOR(conn.version) >= 2)
+        conn.to
+            << options.maxLogSize;
+    if (GET_PROTOCOL_MINOR(conn.version) >= 3)
+        conn.to
+            << options.nrRepeats
+            << options.enforceDeterminism;
+
+    if (GET_PROTOCOL_MINOR(conn.version) >= 7) {
+        conn.to << ((int) options.keepFailed);
+    }
+}
+
 }
diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh
index ada67a149..1665b935f 100644
--- a/src/libstore/serve-protocol.hh
+++ b/src/libstore/serve-protocol.hh
@@ -87,6 +87,13 @@ struct ServeProto
     {
         ServeProto::Serialise<T>::write(store, conn, t);
     }
+
+    /**
+     * Options for building shared between
+     * `ServeProto::Command::BuildPaths` and
+     * `ServeProto::Command::BuildDerivation`.
+     */
+    struct BuildOptions;
 };
 
 enum struct ServeProto::Command : uint64_t
@@ -102,6 +109,22 @@ enum struct ServeProto::Command : uint64_t
     AddToStoreNar = 9,
 };
 
+
+struct ServeProto::BuildOptions {
+    /**
+     * Default value in this and every other field is so tests pass when
+     * testing older deserialisers which do not set all the fields.
+     */
+    time_t maxSilentTime = -1;
+    time_t buildTimeout = -1;
+    size_t maxLogSize = -1;
+    size_t nrRepeats = -1;
+    bool enforceDeterminism = -1;
+    bool keepFailed = -1;
+
+    bool operator == (const ServeProto::BuildOptions &) const = default;
+};
+
 /**
  * Convenience for sending operation codes.
  *
@@ -144,6 +167,8 @@ template<>
 DECLARE_SERVE_SERIALISER(BuildResult);
 template<>
 DECLARE_SERVE_SERIALISER(UnkeyedValidPathInfo);
+template<>
+DECLARE_SERVE_SERIALISER(ServeProto::BuildOptions);
 
 template<typename T>
 DECLARE_SERVE_SERIALISER(std::vector<T>);
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 45af7879c..d361dc0ac 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -835,27 +835,33 @@ static void opServe(Strings opFlags, Strings opArgs)
         verbosity = lvlError;
         settings.keepLog = false;
         settings.useSubstitutes = false;
-        settings.maxSilentTime = readInt(in);
-        settings.buildTimeout = readInt(in);
+
+        auto options = ServeProto::Serialise<ServeProto::BuildOptions>::read(*store, rconn);
+
+        // Only certain feilds get initialized based on the protocol
+        // version. This is why not all the code below is unconditional.
+        // See how the serialization logic in
+        // `ServeProto::Serialise<ServeProto::BuildOptions>` matches
+        // these conditions.
+        settings.maxSilentTime = options.maxSilentTime;
+        settings.buildTimeout = options.buildTimeout;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 2)
-            settings.maxLogSize = readNum<unsigned long>(in);
+            settings.maxLogSize = options.maxLogSize;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 3) {
-            auto nrRepeats = readInt(in);
-            if (nrRepeats != 0) {
+            if (options.nrRepeats != 0) {
                 throw Error("client requested repeating builds, but this is not currently implemented");
             }
-            // Ignore 'enforceDeterminism'. It used to be true by
-            // default, but also only never had any effect when
-            // `nrRepeats == 0`.  We have already asserted that
-            // `nrRepeats` in fact is 0, so we can safely ignore this
-            // without doing something other than what the client
-            // asked for.
-            readInt(in);
-
+            // Ignore 'options.enforceDeterminism'.
+            //
+            // It used to be true by default, but also only never had
+            // any effect when `nrRepeats == 0`.  We have already
+            // checked that `nrRepeats` in fact is 0, so we can safely
+            // ignore this without doing something other than what the
+            // client asked for.
             settings.runDiffHook = true;
         }
         if (GET_PROTOCOL_MINOR(clientVersion) >= 7) {
-            settings.keepFailed = (bool) readInt(in);
+            settings.keepFailed = options.keepFailed;
         }
     };
 
diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.1.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.1.bin
new file mode 100644
index 0000000000000000000000000000000000000000..61e1d97286139e43918505b1b953128360d27853
GIT binary patch
literal 16
NcmZQ&fB-fq4FCX;01N;C

literal 0
HcmV?d00001

diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.2.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.2.bin
new file mode 100644
index 0000000000000000000000000000000000000000..045c2ff2b54ba708bc1d411f0e8786207c4e660a
GIT binary patch
literal 24
PcmZQ&fB-fq%?_mj0Vn_y

literal 0
HcmV?d00001

diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.3.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.3.bin
new file mode 100644
index 0000000000000000000000000000000000000000..5c53458831dca70d5303363919f46f20f88993a2
GIT binary patch
literal 40
VcmZQ&fB-fq%?_nGpfn?t1^@!!02}}S

literal 0
HcmV?d00001

diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.7.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.7.bin
new file mode 100644
index 0000000000000000000000000000000000000000..1bc7b02db38f5f751c2610de84ff937e630567c9
GIT binary patch
literal 48
WcmZQ&fB-fq%?_nGpfrqPgfajFxBwgg

literal 0
HcmV?d00001

diff --git a/tests/unit/libstore/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc
index c2298c6db..8f256d1e6 100644
--- a/tests/unit/libstore/serve-protocol.cc
+++ b/tests/unit/libstore/serve-protocol.cc
@@ -302,6 +302,54 @@ VERSIONED_CHARACTERIZATION_TEST(
         }),
     }))
 
+VERSIONED_CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    build_options_2_1,
+    "build-options-2.1",
+    2 << 8 | 1,
+    (ServeProto::BuildOptions {
+        .maxSilentTime = 5,
+        .buildTimeout = 6,
+    }))
+
+VERSIONED_CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    build_options_2_2,
+    "build-options-2.2",
+    2 << 8 | 2,
+    (ServeProto::BuildOptions {
+        .maxSilentTime = 5,
+        .buildTimeout = 6,
+        .maxLogSize = 7,
+    }))
+
+VERSIONED_CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    build_options_2_3,
+    "build-options-2.3",
+    2 << 8 | 3,
+    (ServeProto::BuildOptions {
+        .maxSilentTime = 5,
+        .buildTimeout = 6,
+        .maxLogSize = 7,
+        .nrRepeats = 8,
+        .enforceDeterminism = true,
+    }))
+
+VERSIONED_CHARACTERIZATION_TEST(
+    ServeProtoTest,
+    build_options_2_7,
+    "build-options-2.7",
+    2 << 8 | 7,
+    (ServeProto::BuildOptions {
+        .maxSilentTime = 5,
+        .buildTimeout = 6,
+        .maxLogSize = 7,
+        .nrRepeats = 8,
+        .enforceDeterminism = false,
+        .keepFailed = true,
+    }))
+
 VERSIONED_CHARACTERIZATION_TEST(
     ServeProtoTest,
     vector,

From 360f3b3a9e0a74eb8b7d5a1744ad58f4cd487ca0 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sat, 9 Dec 2023 19:50:33 +0100
Subject: [PATCH 196/421] changelog-d: Use roberth fork with markdown
 frontmatter support

---
 flake.nix                  |  2 +-
 misc/changelog-d.cabal.nix | 31 +++++++++++++++++++++++++++++++
 misc/changelog-d.nix       | 31 +++++++++++++++++++++++++++++++
 3 files changed, 63 insertions(+), 1 deletion(-)
 create mode 100644 misc/changelog-d.cabal.nix
 create mode 100644 misc/changelog-d.nix

diff --git a/flake.nix b/flake.nix
index dbd45f053..90203e7d5 100644
--- a/flake.nix
+++ b/flake.nix
@@ -182,7 +182,7 @@
           "--enable-internal-api-docs"
         ];
 
-        changelog-d = pkgs.buildPackages.changelog-d;
+        changelog-d = pkgs.buildPackages.callPackage ./misc/changelog-d.nix { };
 
         nativeBuildDeps =
           [
diff --git a/misc/changelog-d.cabal.nix b/misc/changelog-d.cabal.nix
new file mode 100644
index 000000000..76f9353cd
--- /dev/null
+++ b/misc/changelog-d.cabal.nix
@@ -0,0 +1,31 @@
+{ mkDerivation, aeson, base, bytestring, cabal-install-parsers
+, Cabal-syntax, containers, directory, filepath, frontmatter
+, generic-lens-lite, lib, mtl, optparse-applicative, parsec, pretty
+, regex-applicative, text, pkgs
+}:
+let rev = "f30f6969e9cd8b56242309639d58acea21c99d06";
+in
+mkDerivation {
+  pname = "changelog-d";
+  version = "0.1";
+  src = pkgs.fetchurl {
+    name = "changelog-d-${rev}.tar.gz";
+    url = "https://codeberg.org/roberth/changelog-d/archive/${rev}.tar.gz";
+    hash = "sha256-8a2+i5u7YoszAgd5OIEW0eYUcP8yfhtoOIhLJkylYJ4=";
+  } // { inherit rev; };
+  isLibrary = false;
+  isExecutable = true;
+  libraryHaskellDepends = [
+    aeson base bytestring cabal-install-parsers Cabal-syntax containers
+    directory filepath frontmatter generic-lens-lite mtl parsec pretty
+    regex-applicative text
+  ];
+  executableHaskellDepends = [
+    base bytestring Cabal-syntax directory filepath
+    optparse-applicative
+  ];
+  doHaddock = false;
+  description = "Concatenate changelog entries into a single one";
+  license = lib.licenses.gpl3Plus;
+  mainProgram = "changelog-d";
+}
diff --git a/misc/changelog-d.nix b/misc/changelog-d.nix
new file mode 100644
index 000000000..1b20f4596
--- /dev/null
+++ b/misc/changelog-d.nix
@@ -0,0 +1,31 @@
+# Taken temporarily from <nixpkgs/pkgs/by-name/ch/changelog-d/package.nix>
+{
+  callPackage,
+  lib,
+  haskell,
+  haskellPackages,
+}:
+
+let
+  hsPkg = haskellPackages.callPackage ./changelog-d.cabal.nix { };
+
+  addCompletions = haskellPackages.generateOptparseApplicativeCompletions ["changelog-d"];
+
+  haskellModifications =
+    lib.flip lib.pipe [
+      addCompletions
+      haskell.lib.justStaticExecutables
+    ];
+
+  mkDerivationOverrides = finalAttrs: oldAttrs: {
+
+    version = oldAttrs.version + "-git-${lib.strings.substring 0 7 oldAttrs.src.rev}";
+
+    meta = oldAttrs.meta // {
+      homepage = "https://codeberg.org/roberth/changelog-d";
+      maintainers = [ lib.maintainers.roberth ];
+    };
+
+  };
+in
+  (haskellModifications hsPkg).overrideAttrs mkDerivationOverrides

From 3811b334c646bc3b4bf8caef6d13c9f5027246f1 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sat, 9 Dec 2023 19:51:20 +0100
Subject: [PATCH 197/421] rl-next: Use markdown frontmatter syntax

The old syntax is still supported, as long as you don't use a {
in the description - the reason to migrate.
---
 doc/manual/rl-next/hash-format-nix32.md          |  5 +++--
 doc/manual/rl-next/mounted-ssh-store.md          |  9 ++++-----
 doc/manual/rl-next/nix-config-show.md            | 11 +++++------
 doc/manual/rl-next/nix-env-json-drv-path.md      |  9 +++------
 doc/manual/rl-next/nix-hash-convert.md           |  6 +++---
 doc/manual/rl-next/source-positions-in-errors.md |  9 ++++-----
 doc/manual/src/contributing/hacking.md           |  9 ++++-----
 7 files changed, 26 insertions(+), 32 deletions(-)

diff --git a/doc/manual/rl-next/hash-format-nix32.md b/doc/manual/rl-next/hash-format-nix32.md
index 20c557da9..73e6fbb24 100644
--- a/doc/manual/rl-next/hash-format-nix32.md
+++ b/doc/manual/rl-next/hash-format-nix32.md
@@ -1,6 +1,7 @@
+---
 synopsis: Rename hash format `base32` to `nix32`
-prs: #9452
-description: {
+prs: 9452
+---
 
 Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for 
 [Base32](https://en.wikipedia.org/wiki/Base32).
diff --git a/doc/manual/rl-next/mounted-ssh-store.md b/doc/manual/rl-next/mounted-ssh-store.md
index 39fac5283..6df44dbb6 100644
--- a/doc/manual/rl-next/mounted-ssh-store.md
+++ b/doc/manual/rl-next/mounted-ssh-store.md
@@ -1,9 +1,8 @@
+---
 synopsis: Mounted SSH Store
-issues: #7890
-prs: #7912
-description: {
+issues: 7890
+prs: 7912
+---
 
 Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
 This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem.
-
-}
diff --git a/doc/manual/rl-next/nix-config-show.md b/doc/manual/rl-next/nix-config-show.md
index b2ad3c666..26b961b76 100644
--- a/doc/manual/rl-next/nix-config-show.md
+++ b/doc/manual/rl-next/nix-config-show.md
@@ -1,8 +1,7 @@
-synopsis: `nix config show`
-issues: #7672
-prs: #9477
-description: {
+---
+synopsis: Rename to `nix config show`
+issues: 7672
+prs: 9477
+---
 
 `nix show-config` was renamed to `nix config show`, and `nix doctor` was renamed to `nix config check`, to be more consistent with the rest of the command-line interface.
-
-}
diff --git a/doc/manual/rl-next/nix-env-json-drv-path.md b/doc/manual/rl-next/nix-env-json-drv-path.md
index fbe2b67d8..734cefd1b 100644
--- a/doc/manual/rl-next/nix-env-json-drv-path.md
+++ b/doc/manual/rl-next/nix-env-json-drv-path.md
@@ -1,9 +1,6 @@
+---
 synopsis: Fix `nix-env --query --drv-path --json`
-prs: #9257
-description: {
+prs: 9257
+---
 
 Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.
-
-}
-
-
diff --git a/doc/manual/rl-next/nix-hash-convert.md b/doc/manual/rl-next/nix-hash-convert.md
index de4367c5b..2b718a66b 100644
--- a/doc/manual/rl-next/nix-hash-convert.md
+++ b/doc/manual/rl-next/nix-hash-convert.md
@@ -1,6 +1,7 @@
+---
 synopsis: Add `nix hash convert`
-prs: #9452
-description: {
+prs: 9452
+---
 
 New [`nix hash convert`](https://github.com/NixOS/nix/issues/8876) sub command with a fast track
 to stabilization! Examples:
@@ -44,4 +45,3 @@ The following commands are still available but will emit a deprecation warning.
 - `nix hash to-base64 $hash1 $hash2`: Use `nix hash convert --to base64 $hash1 $hash2` instead.
 - `nix hash to-sri $hash1 $hash2`: : Use `nix hash convert --to sri $hash1 $hash2`
   or even just `nix hash convert $hash1 $hash2` instead.
-}
diff --git a/doc/manual/rl-next/source-positions-in-errors.md b/doc/manual/rl-next/source-positions-in-errors.md
index 15df884ea..5b210289d 100644
--- a/doc/manual/rl-next/source-positions-in-errors.md
+++ b/doc/manual/rl-next/source-positions-in-errors.md
@@ -1,7 +1,8 @@
+---
 synopsis: Source locations are printed more consistently in errors
-issues: #561
-prs: #9555
-description: {
+issues: 561
+prs: 9555
+---
 
 Source location information is now included in error messages more
 consistently. Given this code:
@@ -39,5 +40,3 @@ error:
 
        error: value is a set while a string was expected
 ```
-
-}
diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index 9de5ad39b..237eff925 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -257,17 +257,16 @@ User-visible changes should come with a release note.
 Here's what a complete entry looks like. The file name is not incorporated in the document.
 
 ```
+---
 synopsis: Basically a title
-issues: #1234
-prs: #1238
-description: {
+issues: 1234
+prs: 1238
+---
 
 Here's one or more paragraphs that describe the change.
 
 - It's markdown
 - Add references to the manual using @docroot@
-
-}
 ```
 
 Significant changes should add the following header, which moves them to the top.

From a856f603ed5a124f7eb818dadab6c88da73570fb Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sat, 9 Dec 2023 19:55:47 +0100
Subject: [PATCH 198/421] Add checks.rl-next

---
 flake.nix | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/flake.nix b/flake.nix
index 90203e7d5..f499b0a9b 100644
--- a/flake.nix
+++ b/flake.nix
@@ -691,6 +691,11 @@
         perlBindings = self.hydraJobs.perlBindings.${system};
         installTests = self.hydraJobs.installTests.${system};
         nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system};
+        rl-next =
+          let pkgs = nixpkgsFor.${system}.native;
+          in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } ''
+          LANG=C.UTF-8 ${(commonDeps { inherit pkgs; }).changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out
+        '';
       } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
         dockerImage = self.hydraJobs.dockerImage.${system};
       });

From a63be6578f7e17182fdec8e3d3fdbab19a814152 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sat, 9 Dec 2023 21:22:20 +0100
Subject: [PATCH 199/421] flake.nix: Cache shell inputs through hydra

---
 flake.nix | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/flake.nix b/flake.nix
index f499b0a9b..99480183a 100644
--- a/flake.nix
+++ b/flake.nix
@@ -540,6 +540,8 @@
         # Binary package for various platforms.
         build = forAllSystems (system: self.packages.${system}.nix);
 
+        shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation);
+
         buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static);
 
         buildCross = forAllCrossSystems (crossSystem:

From 3c200da242d8f0ccda447866028bb757e0b0bbd9 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Sun, 10 Dec 2023 06:16:32 +0100
Subject: [PATCH 200/421] document `fetchTree` (#9258)

* document `fetchTree`

* display experimental feature note at the top

we have to enable the new `fetchTree` experimental feature to render it
at all. this was a bug introduced when adding that new feature flag.

Co-authored-by: tomberek <tomberek@users.noreply.github.com>
Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
Co-authored-by: Silvan Mosberger <github@infinisil.com>
---
 doc/manual/generate-builtins.nix   |  13 +-
 doc/manual/generate-settings.nix   |   4 +-
 doc/manual/generate-store-info.nix |   4 +-
 src/libexpr/primops/fetchTree.cc   | 242 ++++++++++++++++++++++++-----
 src/nix/main.cc                    |   1 +
 5 files changed, 221 insertions(+), 43 deletions(-)

diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix
index 05cae1c46..007b698f1 100644
--- a/doc/manual/generate-builtins.nix
+++ b/doc/manual/generate-builtins.nix
@@ -8,7 +8,15 @@ let
   showBuiltin = name: { doc, args, arity, experimental-feature }:
     let
       experimentalNotice = optionalString (experimental-feature != null) ''
-        This function is only available if the [${experimental-feature}](@docroot@/contributing/experimental-features.md#xp-feature-${experimental-feature}) experimental feature is enabled.
+        > **Note**
+        >
+        > This function is only available if the [`${experimental-feature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimental-feature}) is enabled.
+        >
+        > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md):
+        >
+        > ```
+        > extra-experimental-features = ${experimental-feature}
+        > ```
       '';
     in
     squash ''
@@ -17,10 +25,9 @@ let
       </dt>
       <dd>
 
-      ${doc}
-
       ${experimentalNotice}
 
+      ${doc}
       </dd>
     '';
   listArgs = args: concatStringsSep " " (map (s: "<var>${s}</var>") args);
diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix
index 74446b70b..504cda362 100644
--- a/doc/manual/generate-settings.nix
+++ b/doc/manual/generate-settings.nix
@@ -20,10 +20,10 @@ let
         else "`${setting}`";
       # separate body to cleanly handle indentation
       body = ''
-          ${description}
-
           ${experimentalFeatureNote}
 
+          ${description}
+
           **Default:** ${showDefault documentDefault defaultValue}
 
           ${showAliases aliases}
diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix
index 57247a181..c311c3c39 100644
--- a/doc/manual/generate-store-info.nix
+++ b/doc/manual/generate-store-info.nix
@@ -19,10 +19,10 @@ let
       result = squash ''
         # ${name}
 
-        ${doc}
-
         ${experimentalFeatureNote}
 
+        ${doc}
+
         ## Settings
 
         ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings}
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 15f870a95..eb2df8626 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -187,45 +187,215 @@ static RegisterPrimOp primop_fetchTree({
     .name = "fetchTree",
     .args = {"input"},
     .doc = R"(
-      Fetch a source tree or a plain file using one of the supported backends.
-      *input* must be a [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references), either in attribute set representation or in the URL-like syntax.
-      The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is enabled.
+      Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
 
-      > **Note**
+      - the resulting fixed-output [store path](@docroot@/glossary.md#gloss-store-path)
+      - the corresponding [NAR](@docroot@/glossary.md#gloss-nar) hash
+      - backend-specific metadata (currently not documented). <!-- TODO: document output attributes -->
+
+      *input* must be an attribute set with the following attributes:
+
+      - `type` (String, required)
+
+        One of the [supported source types](#source-types).
+        This determines other required and allowed input attributes.
+
+      - `narHash` (String, optional)
+
+        The `narHash` parameter can be used to substitute the source of the tree.
+        It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism.
+        If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available.
+
+      A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again.
+      That is, `fetchTree` is idempotent.
+
+      Downloads are cached in `$XDG_CACHE_HOME/nix`.
+      The remote source will be fetched from the network if both are true:
+      - A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store
+
+        > **Note**
+        >
+        > [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching.
+
+      - There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl)
+
+      ## Source types
+
+      The following source types and associated input attributes are supported.
+
+      <!-- TODO: It would be soooo much more predictable to work with (and
+      document) if `fetchTree` was a curried call with the first paramter for
+      `type` or an attribute like `builtins.fetchTree.git`! -->
+
+      - `"file"`
+
+        Place a plain file into the Nix store.
+        This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
+
+        - `url` (String, required)
+
+          Supported protocols:
+
+          - `https`
+
+            > **Example**
+            >
+            > ```nix
+            > fetchTree {
+            >   type = "file";
+            >   url = "https://example.com/index.html";
+            > }
+            > ```
+
+          - `http`
+
+            Insecure HTTP transfer for legacy sources.
+
+            > **Warning**
+            >
+            > HTTP performs no encryption or authentication.
+            > Use a `narHash` known in advance to ensure the output has expected contents.
+
+          - `file`
+
+            A file on the local file system.
+
+            > **Example**
+            >
+            > ```nix
+            > fetchTree {
+            >   type = "file";
+            >   url = "file:///home/eelco/nix/README.md";
+            > }
+            > ```
+
+      - `"tarball"`
+
+        Download a tar archive and extract it into the Nix store.
+        This has the same underyling implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
+
+        - `url` (String, required)
+
+           > **Example**
+           >
+           > ```nix
+           > fetchTree {
+           >   type = "tarball";
+           >   url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
+           > }
+           > ```
+
+      - `"git"`
+
+        Fetch a Git tree and copy it to the Nix store.
+        This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
+
+        - `url` (String, required)
+
+          The URL formats supported are the same as for Git itself.
+
+          > **Example**
+          >
+          > ```nix
+          > fetchTree {
+          >   type = "git";
+          >   url = "git@github.com:NixOS/nixpkgs.git";
+          > }
+          > ```
+
+          > **Note**
+          >
+          > If the URL points to a local directory, and no `ref` or `rev` is given, Nix will only consider files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory.
+
+        - `ref` (String, optional)
+
+          A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
+
+          Default: `"HEAD"`
+
+        - `rev` (String, optional)
+
+          A Git revision; a commit hash.
+
+          Default: the tip of `ref`
+
+        - `shallow` (Bool, optional)
+
+          Make a shallow clone when fetching the Git tree.
+
+          Default: `false`
+
+        - `submodules` (Bool, optional)
+
+          Also fetch submodules if available.
+
+          Default: `false`
+
+        - `allRefs` (Bool, optional)
+
+          If set to `true`, always fetch the entire repository, even if the latest commit is still in the cache.
+          Otherwise, only the latest commit is fetched if it is not already cached.
+
+          Default: `false`
+
+        - `lastModified` (Integer, optional)
+
+          Unix timestamp of the fetched commit.
+
+          If set, pass through the value to the output attribute set.
+          Otherwise, generated from the fetched Git tree.
+
+        - `revCount` (Integer, optional)
+
+          Number of revisions in the history of the Git repository before the fetched commit.
+
+          If set, pass through the value to the output attribute set.
+          Otherwise, generated from the fetched Git tree.
+
+      The following input types are still subject to change:
+
+      - `"path"`
+      - `"github"`
+      - `"gitlab"`
+      - `"sourcehut"`
+      - `"mercurial"`
+
+     *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references).
+     The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled.
+
+      > **Example**
       >
-      > The URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled.
+      > Fetch a GitHub repository using the attribute set representation:
+      >
+      > ```nix
+      > builtins.fetchTree {
+      >   type = "github";
+      >   owner = "NixOS";
+      >   repo = "nixpkgs";
+      >   rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
+      > }
+      > ```
+      >
+      > This evaluates to the following attribute set:
+      >
+      > ```nix
+      > {
+      >   lastModified = 1686503798;
+      >   lastModifiedDate = "20230611171638";
+      >   narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc=";
+      >   outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source";
+      >   rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
+      >   shortRev = "ae2e6b3";
+      > }
+      > ```
 
-      Here are some examples of how to use `fetchTree`:
-
-      - Fetch a GitHub repository using the attribute set representation:
-
-          ```nix
-          builtins.fetchTree {
-            type = "github";
-            owner = "NixOS";
-            repo = "nixpkgs";
-            rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
-          }
-          ```
-
-        This evaluates to the following attribute set:
-
-          ```
-          {
-            lastModified = 1686503798;
-            lastModifiedDate = "20230611171638";
-            narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc=";
-            outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source";
-            rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
-            shortRev = "ae2e6b3";
-          }
-          ```
-
-      - Fetch the same GitHub repository using the URL-like syntax:
-
-          ```
-          builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
-          ```
+      > **Example**
+      >
+      > Fetch the same GitHub repository using the URL-like syntax:
+      >
+      >   ```nix
+      >   builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
+      >   ```
     )",
     .fun = prim_fetchTree,
     .experimentalFeature = Xp::FetchTree,
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 109d2cc04..39c04069b 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -373,6 +373,7 @@ void mainWrapped(int argc, char * * argv)
             Xp::Flakes,
             Xp::FetchClosure,
             Xp::DynamicDerivations,
+            Xp::FetchTree,
         };
         evalSettings.pureEval = false;
         EvalState state({}, openStore("dummy://"));

From deadb3bfe9cde3e78e8e89340e4c92499069461a Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 10 Dec 2023 14:28:14 -0500
Subject: [PATCH 201/421] Create header for `LegacySSHStore`

In https://github.com/NixOS/nix/pull/6134#issuecomment-1079199888,
@thuffschmitt proposed exposing `LegacySSHStore` in Nix for
deduplication with Hydra, at least temporarily. I think that is a good
idea.

Note that the diff will look bad unless one ignores whitespace! Also try
this locally:

```shell-session
git diff --ignore-all-space HEAD^:src/libstore/legacy-ssh-store.cc HEAD:src/libstore/legacy-ssh-store.cc
git diff --ignore-all-space HEAD^:src/libstore/legacy-ssh-store.cc HEAD:src/libstore/legacy-ssh-store.hh
```
---
 src/libstore/legacy-ssh-store.cc | 726 ++++++++++++++-----------------
 src/libstore/legacy-ssh-store.hh | 132 ++++++
 2 files changed, 466 insertions(+), 392 deletions(-)
 create mode 100644 src/libstore/legacy-ssh-store.hh

diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 8ef2daa7b..06bef9d08 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -1,3 +1,4 @@
+#include "legacy-ssh-store.hh"
 #include "ssh-store-config.hh"
 #include "archive.hh"
 #include "pool.hh"
@@ -13,414 +14,355 @@
 
 namespace nix {
 
-struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
+std::string LegacySSHStoreConfig::doc()
 {
-    using CommonSSHStoreConfig::CommonSSHStoreConfig;
+    return
+      #include "legacy-ssh-store.md"
+      ;
+}
 
-    const Setting<Path> remoteProgram{this, "nix-store", "remote-program",
-        "Path to the `nix-store` executable on the remote machine."};
 
-    const Setting<int> maxConnections{this, 1, "max-connections",
-        "Maximum number of concurrent SSH connections."};
-
-    const std::string name() override { return "SSH Store"; }
-
-    std::string doc() override
-    {
-        return
-          #include "legacy-ssh-store.md"
-          ;
-    }
-};
-
-struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store
+struct LegacySSHStore::Connection
 {
-    // Hack for getting remote build log output.
-    // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
-    // the documentation
-    const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
-
-    struct Connection
-    {
-        std::unique_ptr<SSHMaster::Connection> sshConn;
-        FdSink to;
-        FdSource from;
-        ServeProto::Version remoteVersion;
-        bool good = true;
-
-        /**
-         * Coercion to `ServeProto::ReadConn`. This makes it easy to use the
-         * factored out serve protocol searlizers with a
-         * `LegacySSHStore::Connection`.
-         *
-         * The serve protocol connection types are unidirectional, unlike
-         * this type.
-         */
-        operator ServeProto::ReadConn ()
-        {
-            return ServeProto::ReadConn {
-                .from = from,
-                .version = remoteVersion,
-            };
-        }
-
-        /*
-         * Coercion to `ServeProto::WriteConn`. This makes it easy to use the
-         * factored out serve protocol searlizers with a
-         * `LegacySSHStore::Connection`.
-         *
-         * The serve protocol connection types are unidirectional, unlike
-         * this type.
-         */
-        operator ServeProto::WriteConn ()
-        {
-            return ServeProto::WriteConn {
-                .to = to,
-                .version = remoteVersion,
-            };
-        }
-    };
-
-    std::string host;
-
-    ref<Pool<Connection>> connections;
-
-    SSHMaster master;
-
-    static std::set<std::string> uriSchemes() { return {"ssh"}; }
-
-    LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params)
-        : StoreConfig(params)
-        , CommonSSHStoreConfig(params)
-        , LegacySSHStoreConfig(params)
-        , Store(params)
-        , host(host)
-        , connections(make_ref<Pool<Connection>>(
-            std::max(1, (int) maxConnections),
-            [this]() { return openConnection(); },
-            [](const ref<Connection> & r) { return r->good; }
-            ))
-        , master(
-            host,
-            sshKey,
-            sshPublicHostKey,
-            // Use SSH master only if using more than 1 connection.
-            connections->capacity() > 1,
-            compress,
-            logFD)
-    {
-    }
-
-    ref<Connection> openConnection()
-    {
-        auto conn = make_ref<Connection>();
-        conn->sshConn = master.startCommand(
-            fmt("%s --serve --write", remoteProgram)
-            + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
-        conn->to = FdSink(conn->sshConn->in.get());
-        conn->from = FdSource(conn->sshConn->out.get());
-
-        try {
-            conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION;
-            conn->to.flush();
-
-            StringSink saved;
-            try {
-                TeeSource tee(conn->from, saved);
-                unsigned int magic = readInt(tee);
-                if (magic != SERVE_MAGIC_2)
-                    throw Error("'nix-store --serve' protocol mismatch from '%s'", host);
-            } catch (SerialisationError & e) {
-                /* In case the other side is waiting for our input,
-                   close it. */
-                conn->sshConn->in.close();
-                auto msg = conn->from.drain();
-                throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'",
-                    host, chomp(saved.s + msg));
-            }
-            conn->remoteVersion = readInt(conn->from);
-            if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200)
-                throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host);
-
-        } catch (EndOfFile & e) {
-            throw Error("cannot connect to '%1%'", host);
-        }
-
-        return conn;
-    };
-
-    std::string getUri() override
-    {
-        return *uriSchemes().begin() + "://" + host;
-    }
-
-    void queryPathInfoUncached(const StorePath & path,
-        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
-    {
-        try {
-            auto conn(connections->get());
-
-            /* No longer support missing NAR hash */
-            assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4);
-
-            debug("querying remote host '%s' for info on '%s'", host, printStorePath(path));
-
-            conn->to << ServeProto::Command::QueryPathInfos << PathSet{printStorePath(path)};
-            conn->to.flush();
-
-            auto p = readString(conn->from);
-            if (p.empty()) return callback(nullptr);
-            auto path2 = parseStorePath(p);
-            assert(path == path2);
-            auto info = std::make_shared<ValidPathInfo>(
-                path,
-                ServeProto::Serialise<UnkeyedValidPathInfo>::read(*this, *conn));
-
-            if (info->narHash == Hash::dummy)
-                throw Error("NAR hash is now mandatory");
-
-            auto s = readString(conn->from);
-            assert(s == "");
-
-            callback(std::move(info));
-        } catch (...) { callback.rethrow(); }
-    }
-
-    void addToStore(const ValidPathInfo & info, Source & source,
-        RepairFlag repair, CheckSigsFlag checkSigs) override
-    {
-        debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host);
-
-        auto conn(connections->get());
-
-        if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) {
-
-            conn->to
-                << ServeProto::Command::AddToStoreNar
-                << printStorePath(info.path)
-                << (info.deriver ? printStorePath(*info.deriver) : "")
-                << info.narHash.to_string(HashFormat::Base16, false);
-            ServeProto::write(*this, *conn, info.references);
-            conn->to
-                << info.registrationTime
-                << info.narSize
-                << info.ultimate
-                << info.sigs
-                << renderContentAddress(info.ca);
-            try {
-                copyNAR(source, conn->to);
-            } catch (...) {
-                conn->good = false;
-                throw;
-            }
-            conn->to.flush();
-
-        } else {
-
-            conn->to
-                << ServeProto::Command::ImportPaths
-                << 1;
-            try {
-                copyNAR(source, conn->to);
-            } catch (...) {
-                conn->good = false;
-                throw;
-            }
-            conn->to
-                << exportMagic
-                << printStorePath(info.path);
-            ServeProto::write(*this, *conn, info.references);
-            conn->to
-                << (info.deriver ? printStorePath(*info.deriver) : "")
-                << 0
-                << 0;
-            conn->to.flush();
-
-        }
-
-        if (readInt(conn->from) != 1)
-            throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host);
-    }
-
-    void narFromPath(const StorePath & path, Sink & sink) override
-    {
-        auto conn(connections->get());
-
-        conn->to << ServeProto::Command::DumpStorePath << printStorePath(path);
-        conn->to.flush();
-        copyNAR(conn->from, sink);
-    }
-
-    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
-    { unsupported("queryPathFromHashPart"); }
-
-    StorePath addToStore(
-            std::string_view name,
-            const Path & srcPath,
-            FileIngestionMethod method,
-            HashAlgorithm hashAlgo,
-            PathFilter & filter,
-            RepairFlag repair,
-            const StorePathSet & references) override
-    { unsupported("addToStore"); }
-
-    StorePath addTextToStore(
-        std::string_view name,
-        std::string_view s,
-        const StorePathSet & references,
-        RepairFlag repair) override
-    { unsupported("addTextToStore"); }
-
-private:
-
-    void putBuildSettings(Connection & conn)
-    {
-        ServeProto::write(*this, conn, ServeProto::BuildOptions {
-            .maxSilentTime = settings.maxSilentTime,
-            .buildTimeout = settings.buildTimeout,
-            .maxLogSize = settings.maxLogSize,
-            .nrRepeats = 0, // buildRepeat hasn't worked for ages anyway
-            .enforceDeterminism = 0,
-            .keepFailed = settings.keepFailed,
-        });
-    }
-
-public:
-
-    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
-        BuildMode buildMode) override
-    {
-        auto conn(connections->get());
-
-        conn->to
-            << ServeProto::Command::BuildDerivation
-            << printStorePath(drvPath);
-        writeDerivation(conn->to, *this, drv);
-
-        putBuildSettings(*conn);
-
-        conn->to.flush();
-
-        return ServeProto::Serialise<BuildResult>::read(*this, *conn);
-    }
-
-    void buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override
-    {
-        if (evalStore && evalStore.get() != this)
-            throw Error("building on an SSH store is incompatible with '--eval-store'");
-
-        auto conn(connections->get());
-
-        conn->to << ServeProto::Command::BuildPaths;
-        Strings ss;
-        for (auto & p : drvPaths) {
-            auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p);
-            std::visit(overloaded {
-                [&](const StorePathWithOutputs & s) {
-                    ss.push_back(s.to_string(*this));
-                },
-                [&](const StorePath & drvPath) {
-                    throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath));
-                },
-                [&](std::monostate) {
-                    throw Error("wanted build derivation that is itself a build product, but the legacy ssh protocol doesn't support that. Try using ssh-ng://");
-                },
-            }, sOrDrvPath);
-        }
-        conn->to << ss;
-
-        putBuildSettings(*conn);
-
-        conn->to.flush();
-
-        BuildResult result;
-        result.status = (BuildResult::Status) readInt(conn->from);
-
-        if (!result.success()) {
-            conn->from >> result.errorMsg;
-            throw Error(result.status, result.errorMsg);
-        }
-    }
-
-    void ensurePath(const StorePath & path) override
-    { unsupported("ensurePath"); }
-
-    virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
-    { unsupported("getFSAccessor"); }
+    std::unique_ptr<SSHMaster::Connection> sshConn;
+    FdSink to;
+    FdSource from;
+    ServeProto::Version remoteVersion;
+    bool good = true;
 
     /**
-     * The default instance would schedule the work on the client side, but
-     * for consistency with `buildPaths` and `buildDerivation` it should happen
-     * on the remote side.
+     * Coercion to `ServeProto::ReadConn`. This makes it easy to use the
+     * factored out serve protocol searlizers with a
+     * `LegacySSHStore::Connection`.
      *
-     * We make this fail for now so we can add implement this properly later
-     * without it being a breaking change.
+     * The serve protocol connection types are unidirectional, unlike
+     * this type.
      */
-    void repairPath(const StorePath & path) override
-    { unsupported("repairPath"); }
-
-    void computeFSClosure(const StorePathSet & paths,
-        StorePathSet & out, bool flipDirection = false,
-        bool includeOutputs = false, bool includeDerivers = false) override
+    operator ServeProto::ReadConn ()
     {
-        if (flipDirection || includeDerivers) {
-            Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers);
-            return;
-        }
-
-        auto conn(connections->get());
-
-        conn->to
-            << ServeProto::Command::QueryClosure
-            << includeOutputs;
-        ServeProto::write(*this, *conn, paths);
-        conn->to.flush();
-
-        for (auto & i : ServeProto::Serialise<StorePathSet>::read(*this, *conn))
-            out.insert(i);
+        return ServeProto::ReadConn {
+            .from = from,
+            .version = remoteVersion,
+        };
     }
 
-    StorePathSet queryValidPaths(const StorePathSet & paths,
-        SubstituteFlag maybeSubstitute = NoSubstitute) override
-    {
-        auto conn(connections->get());
-
-        conn->to
-            << ServeProto::Command::QueryValidPaths
-            << false // lock
-            << maybeSubstitute;
-        ServeProto::write(*this, *conn, paths);
-        conn->to.flush();
-
-        return ServeProto::Serialise<StorePathSet>::read(*this, *conn);
-    }
-
-    void connect() override
-    {
-        auto conn(connections->get());
-    }
-
-    unsigned int getProtocol() override
-    {
-        auto conn(connections->get());
-        return conn->remoteVersion;
-    }
-
-    /**
-     * The legacy ssh protocol doesn't support checking for trusted-user.
-     * Try using ssh-ng:// instead if you want to know.
+    /*
+     * Coercion to `ServeProto::WriteConn`. This makes it easy to use the
+     * factored out serve protocol searlizers with a
+     * `LegacySSHStore::Connection`.
+     *
+     * The serve protocol connection types are unidirectional, unlike
+     * this type.
      */
-    std::optional<TrustedFlag> isTrustedClient() override
+    operator ServeProto::WriteConn ()
     {
-        return std::nullopt;
+        return ServeProto::WriteConn {
+            .to = to,
+            .version = remoteVersion,
+        };
     }
-
-    void queryRealisationUncached(const DrvOutput &,
-        Callback<std::shared_ptr<const Realisation>> callback) noexcept override
-    // TODO: Implement
-    { unsupported("queryRealisation"); }
 };
 
+
+LegacySSHStore::LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params)
+    : StoreConfig(params)
+    , CommonSSHStoreConfig(params)
+    , LegacySSHStoreConfig(params)
+    , Store(params)
+    , host(host)
+    , connections(make_ref<Pool<Connection>>(
+        std::max(1, (int) maxConnections),
+        [this]() { return openConnection(); },
+        [](const ref<Connection> & r) { return r->good; }
+        ))
+    , master(
+        host,
+        sshKey,
+        sshPublicHostKey,
+        // Use SSH master only if using more than 1 connection.
+        connections->capacity() > 1,
+        compress,
+        logFD)
+{
+}
+
+
+ref<LegacySSHStore::Connection> LegacySSHStore::openConnection()
+{
+    auto conn = make_ref<Connection>();
+    conn->sshConn = master.startCommand(
+        fmt("%s --serve --write", remoteProgram)
+        + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
+    conn->to = FdSink(conn->sshConn->in.get());
+    conn->from = FdSource(conn->sshConn->out.get());
+
+    try {
+        conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION;
+        conn->to.flush();
+
+        StringSink saved;
+        try {
+            TeeSource tee(conn->from, saved);
+            unsigned int magic = readInt(tee);
+            if (magic != SERVE_MAGIC_2)
+                throw Error("'nix-store --serve' protocol mismatch from '%s'", host);
+        } catch (SerialisationError & e) {
+            /* In case the other side is waiting for our input,
+               close it. */
+            conn->sshConn->in.close();
+            auto msg = conn->from.drain();
+            throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'",
+                host, chomp(saved.s + msg));
+        }
+        conn->remoteVersion = readInt(conn->from);
+        if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200)
+            throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host);
+
+    } catch (EndOfFile & e) {
+        throw Error("cannot connect to '%1%'", host);
+    }
+
+    return conn;
+};
+
+
+std::string LegacySSHStore::getUri()
+{
+    return *uriSchemes().begin() + "://" + host;
+}
+
+
+void LegacySSHStore::queryPathInfoUncached(const StorePath & path,
+    Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
+{
+    try {
+        auto conn(connections->get());
+
+        /* No longer support missing NAR hash */
+        assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4);
+
+        debug("querying remote host '%s' for info on '%s'", host, printStorePath(path));
+
+        conn->to << ServeProto::Command::QueryPathInfos << PathSet{printStorePath(path)};
+        conn->to.flush();
+
+        auto p = readString(conn->from);
+        if (p.empty()) return callback(nullptr);
+        auto path2 = parseStorePath(p);
+        assert(path == path2);
+        auto info = std::make_shared<ValidPathInfo>(
+            path,
+            ServeProto::Serialise<UnkeyedValidPathInfo>::read(*this, *conn));
+
+        if (info->narHash == Hash::dummy)
+            throw Error("NAR hash is now mandatory");
+
+        auto s = readString(conn->from);
+        assert(s == "");
+
+        callback(std::move(info));
+    } catch (...) { callback.rethrow(); }
+}
+
+
+void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source,
+    RepairFlag repair, CheckSigsFlag checkSigs)
+{
+    debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host);
+
+    auto conn(connections->get());
+
+    if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) {
+
+        conn->to
+            << ServeProto::Command::AddToStoreNar
+            << printStorePath(info.path)
+            << (info.deriver ? printStorePath(*info.deriver) : "")
+            << info.narHash.to_string(HashFormat::Base16, false);
+        ServeProto::write(*this, *conn, info.references);
+        conn->to
+            << info.registrationTime
+            << info.narSize
+            << info.ultimate
+            << info.sigs
+            << renderContentAddress(info.ca);
+        try {
+            copyNAR(source, conn->to);
+        } catch (...) {
+            conn->good = false;
+            throw;
+        }
+        conn->to.flush();
+
+    } else {
+
+        conn->to
+            << ServeProto::Command::ImportPaths
+            << 1;
+        try {
+            copyNAR(source, conn->to);
+        } catch (...) {
+            conn->good = false;
+            throw;
+        }
+        conn->to
+            << exportMagic
+            << printStorePath(info.path);
+        ServeProto::write(*this, *conn, info.references);
+        conn->to
+            << (info.deriver ? printStorePath(*info.deriver) : "")
+            << 0
+            << 0;
+        conn->to.flush();
+
+    }
+
+    if (readInt(conn->from) != 1)
+        throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host);
+}
+
+
+void LegacySSHStore::narFromPath(const StorePath & path, Sink & sink)
+{
+    auto conn(connections->get());
+
+    conn->to << ServeProto::Command::DumpStorePath << printStorePath(path);
+    conn->to.flush();
+    copyNAR(conn->from, sink);
+}
+
+
+void LegacySSHStore::putBuildSettings(Connection & conn)
+{
+    ServeProto::write(*this, conn, ServeProto::BuildOptions {
+        .maxSilentTime = settings.maxSilentTime,
+        .buildTimeout = settings.buildTimeout,
+        .maxLogSize = settings.maxLogSize,
+        .nrRepeats = 0, // buildRepeat hasn't worked for ages anyway
+        .enforceDeterminism = 0,
+        .keepFailed = settings.keepFailed,
+    });
+}
+
+
+BuildResult LegacySSHStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
+    BuildMode buildMode)
+{
+    auto conn(connections->get());
+
+    conn->to
+        << ServeProto::Command::BuildDerivation
+        << printStorePath(drvPath);
+    writeDerivation(conn->to, *this, drv);
+
+    putBuildSettings(*conn);
+
+    conn->to.flush();
+
+    return ServeProto::Serialise<BuildResult>::read(*this, *conn);
+}
+
+
+void LegacySSHStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore)
+{
+    if (evalStore && evalStore.get() != this)
+        throw Error("building on an SSH store is incompatible with '--eval-store'");
+
+    auto conn(connections->get());
+
+    conn->to << ServeProto::Command::BuildPaths;
+    Strings ss;
+    for (auto & p : drvPaths) {
+        auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p);
+        std::visit(overloaded {
+            [&](const StorePathWithOutputs & s) {
+                ss.push_back(s.to_string(*this));
+            },
+            [&](const StorePath & drvPath) {
+                throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath));
+            },
+            [&](std::monostate) {
+                throw Error("wanted build derivation that is itself a build product, but the legacy ssh protocol doesn't support that. Try using ssh-ng://");
+            },
+        }, sOrDrvPath);
+    }
+    conn->to << ss;
+
+    putBuildSettings(*conn);
+
+    conn->to.flush();
+
+    BuildResult result;
+    result.status = (BuildResult::Status) readInt(conn->from);
+
+    if (!result.success()) {
+        conn->from >> result.errorMsg;
+        throw Error(result.status, result.errorMsg);
+    }
+}
+
+
+void LegacySSHStore::computeFSClosure(const StorePathSet & paths,
+    StorePathSet & out, bool flipDirection,
+    bool includeOutputs, bool includeDerivers)
+{
+    if (flipDirection || includeDerivers) {
+        Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers);
+        return;
+    }
+
+    auto conn(connections->get());
+
+    conn->to
+        << ServeProto::Command::QueryClosure
+        << includeOutputs;
+    ServeProto::write(*this, *conn, paths);
+    conn->to.flush();
+
+    for (auto & i : ServeProto::Serialise<StorePathSet>::read(*this, *conn))
+        out.insert(i);
+}
+
+
+StorePathSet LegacySSHStore::queryValidPaths(const StorePathSet & paths,
+    SubstituteFlag maybeSubstitute)
+{
+    auto conn(connections->get());
+
+    conn->to
+        << ServeProto::Command::QueryValidPaths
+        << false // lock
+        << maybeSubstitute;
+    ServeProto::write(*this, *conn, paths);
+    conn->to.flush();
+
+    return ServeProto::Serialise<StorePathSet>::read(*this, *conn);
+}
+
+
+void LegacySSHStore::connect()
+{
+    auto conn(connections->get());
+}
+
+
+unsigned int LegacySSHStore::getProtocol()
+{
+    auto conn(connections->get());
+    return conn->remoteVersion;
+}
+
+
+/**
+ * The legacy ssh protocol doesn't support checking for trusted-user.
+ * Try using ssh-ng:// instead if you want to know.
+ */
+std::optional<TrustedFlag> isTrustedClient()
+{
+    return std::nullopt;
+}
+
+
 static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regLegacySSHStore;
 
 }
diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh
new file mode 100644
index 000000000..c40c256bb
--- /dev/null
+++ b/src/libstore/legacy-ssh-store.hh
@@ -0,0 +1,132 @@
+#pragma once
+///@file
+
+#include "ssh-store-config.hh"
+#include "store-api.hh"
+#include "ssh.hh"
+#include "callback.hh"
+#include "pool.hh"
+
+namespace nix {
+
+struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
+{
+    using CommonSSHStoreConfig::CommonSSHStoreConfig;
+
+    const Setting<Path> remoteProgram{this, "nix-store", "remote-program",
+        "Path to the `nix-store` executable on the remote machine."};
+
+    const Setting<int> maxConnections{this, 1, "max-connections",
+        "Maximum number of concurrent SSH connections."};
+
+    const std::string name() override { return "SSH Store"; }
+
+    std::string doc() override;
+};
+
+struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store
+{
+    // Hack for getting remote build log output.
+    // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
+    // the documentation
+    const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
+
+    struct Connection;
+
+    std::string host;
+
+    ref<Pool<Connection>> connections;
+
+    SSHMaster master;
+
+    static std::set<std::string> uriSchemes() { return {"ssh"}; }
+
+    LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params);
+
+    ref<Connection> openConnection();
+
+    std::string getUri() override;
+
+    void queryPathInfoUncached(const StorePath & path,
+        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
+
+    void addToStore(const ValidPathInfo & info, Source & source,
+        RepairFlag repair, CheckSigsFlag checkSigs) override;
+
+    void narFromPath(const StorePath & path, Sink & sink) override;
+
+    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
+    { unsupported("queryPathFromHashPart"); }
+
+    StorePath addToStore(
+            std::string_view name,
+            const Path & srcPath,
+            FileIngestionMethod method,
+            HashAlgorithm hashAlgo,
+            PathFilter & filter,
+            RepairFlag repair,
+            const StorePathSet & references) override
+    { unsupported("addToStore"); }
+
+    StorePath addTextToStore(
+        std::string_view name,
+        std::string_view s,
+        const StorePathSet & references,
+        RepairFlag repair) override
+    { unsupported("addTextToStore"); }
+
+private:
+
+    void putBuildSettings(Connection & conn);
+
+public:
+
+    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
+        BuildMode buildMode) override;
+
+    void buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override;
+
+    void ensurePath(const StorePath & path) override
+    { unsupported("ensurePath"); }
+
+    virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
+    { unsupported("getFSAccessor"); }
+
+    /**
+     * The default instance would schedule the work on the client side, but
+     * for consistency with `buildPaths` and `buildDerivation` it should happen
+     * on the remote side.
+     *
+     * We make this fail for now so we can add implement this properly later
+     * without it being a breaking change.
+     */
+    void repairPath(const StorePath & path) override
+    { unsupported("repairPath"); }
+
+    void computeFSClosure(const StorePathSet & paths,
+        StorePathSet & out, bool flipDirection = false,
+        bool includeOutputs = false, bool includeDerivers = false) override;
+
+    StorePathSet queryValidPaths(const StorePathSet & paths,
+        SubstituteFlag maybeSubstitute = NoSubstitute) override;
+
+    void connect() override;
+
+    unsigned int getProtocol() override;
+
+    /**
+     * The legacy ssh protocol doesn't support checking for trusted-user.
+     * Try using ssh-ng:// instead if you want to know.
+     */
+    std::optional<TrustedFlag> isTrustedClient() override
+    {
+        return std::nullopt;
+    }
+
+    void queryRealisationUncached(const DrvOutput &,
+        Callback<std::shared_ptr<const Realisation>> callback) noexcept override
+    // TODO: Implement
+    { unsupported("queryRealisation"); }
+};
+
+}

From e43bb655feaa23977322d68278c88ac075eb2c41 Mon Sep 17 00:00:00 2001
From: Adam Joseph <adam@westernsemico.com>
Date: Sun, 10 Dec 2023 13:58:35 -0800
Subject: [PATCH 202/421] libstore/daemon.cc: note trust model difference in
 readDerivation()s

Below the comment added by this commit is a much longer comment
followed by a trust check, both of which have confused me on at
least two occasions.  I figured it out once, forgot it, then had to
ask @Ericson2314 to explain it, at which point I understood it
again.  I think this might confuse other people too, or maybe I will
just forget it a third time.  So let's add a comment.

Farther down in the function is the following check:

```
if (!(drvType.isCA() || trusted))
  throw Error("you are not privileged to build input-addressed derivations");
```

This seems really strange at first.  A key property of Nix is that
you can compute the outpath of a derivation using the derivation
(and its references-closure) without trusting anybody!

The missing insight is that at this point in the code the builder
doesn't necessarily have the references-closure of the derivation
being built, and therefore needs to trust that the derivation's
outPath is honest.  It's incredibly easy to overlook this, because
the only difference between these two cases is which of these
identically-named functions we used:

- `readDerivation(Source,Store)`
- `Store::readDerivation()`

These functions have different trust models (except in the special
case where the first function is used on the local store).  We
should call the reader's attention to this fact.

Co-authored-by: Cole Helbling <cole.e.helbling@outlook.com>
---
 src/libstore/daemon.cc | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 530b1a178..a112d6d31 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -574,6 +574,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
     case WorkerProto::Op::BuildDerivation: {
         auto drvPath = store->parseStorePath(readString(from));
         BasicDerivation drv;
+        /*
+         * Note: unlike wopEnsurePath, this operation reads a
+         * derivation-to-be-realized from the client with
+         * readDerivation(Source,Store) rather than reading it from
+         * the local store with Store::readDerivation().  Since the
+         * derivation-to-be-realized is not registered in the store
+         * it cannot be trusted that its outPath was calculated
+         * correctly.
+         */
         readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath));
         BuildMode buildMode = (BuildMode) readInt(from);
         logger->startWork();

From 91ba7b230777e3fb023bda48c269d533702e50e8 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Wed, 6 Dec 2023 12:41:47 +0100
Subject: [PATCH 203/421] isAllowedURI: Extract function and test

---
 src/libexpr/eval.cc        |  18 +++++--
 src/libexpr/eval.hh        |   5 ++
 tests/unit/libexpr/eval.cc | 106 +++++++++++++++++++++++++++++++++++++
 3 files changed, 124 insertions(+), 5 deletions(-)
 create mode 100644 tests/unit/libexpr/eval.cc

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 9e494148e..0eb6f406e 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -599,21 +599,29 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value &
     mkStorePathString(storePath, v);
 }
 
-void EvalState::checkURI(const std::string & uri)
+bool isAllowedURI(std::string_view uri, const Strings & allowedUris)
 {
-    if (!evalSettings.restrictEval) return;
-
     /* 'uri' should be equal to a prefix, or in a subdirectory of a
        prefix. Thus, the prefix https://github.co does not permit
        access to https://github.com. Note: this allows 'http://' and
        'https://' as prefixes for any http/https URI. */
-    for (auto & prefix : evalSettings.allowedUris.get())
+    for (auto & prefix : allowedUris) {
         if (uri == prefix ||
             (uri.size() > prefix.size()
             && prefix.size() > 0
             && hasPrefix(uri, prefix)
             && (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/')))
-            return;
+            return true;
+    }
+
+    return false;
+}
+
+void EvalState::checkURI(const std::string & uri)
+{
+    if (!evalSettings.restrictEval) return;
+
+    if (isAllowedURI(uri, evalSettings.allowedUris.get())) return;
 
     /* If the URI is a path, then check it against allowedPaths as
        well. */
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index f3f6d35b9..6008c3f60 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -837,6 +837,11 @@ std::string showType(const Value & v);
  */
 SourcePath resolveExprPath(SourcePath path);
 
+/**
+ * Whether a URI is allowed, assuming restrictEval is enabled
+ */
+bool isAllowedURI(std::string_view uri, const Strings & allowedPaths);
+
 struct InvalidPathError : EvalError
 {
     Path path;
diff --git a/tests/unit/libexpr/eval.cc b/tests/unit/libexpr/eval.cc
new file mode 100644
index 000000000..cc5d6bbfa
--- /dev/null
+++ b/tests/unit/libexpr/eval.cc
@@ -0,0 +1,106 @@
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "eval.hh"
+#include "tests/libexpr.hh"
+
+namespace nix {
+
+TEST(nix_isAllowedURI, http_example_com) {
+    Strings allowed;
+    allowed.push_back("http://example.com");
+
+    ASSERT_TRUE(isAllowedURI("http://example.com", allowed));
+    ASSERT_TRUE(isAllowedURI("http://example.com/foo", allowed));
+    ASSERT_TRUE(isAllowedURI("http://example.com/foo/", allowed));
+    ASSERT_FALSE(isAllowedURI("/", allowed));
+    ASSERT_FALSE(isAllowedURI("http://example.co", allowed));
+    ASSERT_FALSE(isAllowedURI("http://example.como", allowed));
+    ASSERT_FALSE(isAllowedURI("http://example.org", allowed));
+    ASSERT_FALSE(isAllowedURI("http://example.org/foo", allowed));
+}
+
+TEST(nix_isAllowedURI, http_example_com_foo) {
+    Strings allowed;
+    allowed.push_back("http://example.com/foo");
+
+    ASSERT_TRUE(isAllowedURI("http://example.com/foo", allowed));
+    ASSERT_TRUE(isAllowedURI("http://example.com/foo/", allowed));
+    ASSERT_FALSE(isAllowedURI("/foo", allowed));
+    ASSERT_FALSE(isAllowedURI("http://example.com", allowed));
+    ASSERT_FALSE(isAllowedURI("http://example.como", allowed));
+    ASSERT_FALSE(isAllowedURI("http://example.org/foo", allowed));
+    // Broken?
+    // ASSERT_TRUE(isAllowedURI("http://example.com/foo?ok=1", allowed));
+}
+
+TEST(nix_isAllowedURI, http) {
+    Strings allowed;
+    allowed.push_back("http://");
+
+    ASSERT_TRUE(isAllowedURI("http://", allowed));
+    ASSERT_TRUE(isAllowedURI("http://example.com", allowed));
+    ASSERT_TRUE(isAllowedURI("http://example.com/foo", allowed));
+    ASSERT_TRUE(isAllowedURI("http://example.com/foo/", allowed));
+    ASSERT_TRUE(isAllowedURI("http://example.com", allowed));
+    ASSERT_FALSE(isAllowedURI("/", allowed));
+    ASSERT_FALSE(isAllowedURI("https://", allowed));
+    ASSERT_FALSE(isAllowedURI("http:foo", allowed));
+}
+
+TEST(nix_isAllowedURI, https) {
+    Strings allowed;
+    allowed.push_back("https://");
+
+    ASSERT_TRUE(isAllowedURI("https://example.com", allowed));
+    ASSERT_TRUE(isAllowedURI("https://example.com/foo", allowed));
+    ASSERT_FALSE(isAllowedURI("http://example.com", allowed));
+    ASSERT_FALSE(isAllowedURI("http://example.com/https:", allowed));
+}
+
+TEST(nix_isAllowedURI, absolute_path) {
+    Strings allowed;
+    allowed.push_back("/var/evil"); // bad idea
+
+    ASSERT_TRUE(isAllowedURI("/var/evil", allowed));
+    ASSERT_TRUE(isAllowedURI("/var/evil/", allowed));
+    ASSERT_TRUE(isAllowedURI("/var/evil/foo", allowed));
+    ASSERT_TRUE(isAllowedURI("/var/evil/foo/", allowed));
+    ASSERT_FALSE(isAllowedURI("/", allowed));
+    ASSERT_FALSE(isAllowedURI("/var/evi", allowed));
+    ASSERT_FALSE(isAllowedURI("/var/evilo", allowed));
+    ASSERT_FALSE(isAllowedURI("/var/evilo/", allowed));
+    ASSERT_FALSE(isAllowedURI("/var/evilo/foo", allowed));
+    ASSERT_FALSE(isAllowedURI("http://example.com/var/evil", allowed));
+    ASSERT_FALSE(isAllowedURI("http://example.com//var/evil", allowed));
+    ASSERT_FALSE(isAllowedURI("http://example.com//var/evil/foo", allowed));
+}
+
+TEST(nix_isAllowedURI, file_url) {
+    Strings allowed;
+    allowed.push_back("file:///var/evil"); // bad idea
+
+    ASSERT_TRUE(isAllowedURI("file:///var/evil", allowed));
+    ASSERT_TRUE(isAllowedURI("file:///var/evil/", allowed));
+    ASSERT_TRUE(isAllowedURI("file:///var/evil/foo", allowed));
+    ASSERT_TRUE(isAllowedURI("file:///var/evil/foo/", allowed));
+    ASSERT_FALSE(isAllowedURI("/", allowed));
+    ASSERT_FALSE(isAllowedURI("/var/evi", allowed));
+    ASSERT_FALSE(isAllowedURI("/var/evilo", allowed));
+    ASSERT_FALSE(isAllowedURI("/var/evilo/", allowed));
+    ASSERT_FALSE(isAllowedURI("/var/evilo/foo", allowed));
+    ASSERT_FALSE(isAllowedURI("http://example.com/var/evil", allowed));
+    ASSERT_FALSE(isAllowedURI("http://example.com//var/evil", allowed));
+    ASSERT_FALSE(isAllowedURI("http://example.com//var/evil/foo", allowed));
+    ASSERT_FALSE(isAllowedURI("http://var/evil", allowed));
+    ASSERT_FALSE(isAllowedURI("http:///var/evil", allowed));
+    ASSERT_FALSE(isAllowedURI("http://var/evil/", allowed));
+    ASSERT_FALSE(isAllowedURI("file:///var/evi", allowed));
+    ASSERT_FALSE(isAllowedURI("file:///var/evilo", allowed));
+    ASSERT_FALSE(isAllowedURI("file:///var/evilo/", allowed));
+    ASSERT_FALSE(isAllowedURI("file:///var/evilo/foo", allowed));
+    ASSERT_FALSE(isAllowedURI("file:///", allowed));
+    ASSERT_FALSE(isAllowedURI("file://", allowed));
+}
+
+} // namespace nix
\ No newline at end of file

From 6cbba914a70eb5da6447fee5528a63723ed13245 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Wed, 6 Dec 2023 12:43:20 +0100
Subject: [PATCH 204/421] isAllowedURI: Remove incorrect note

---
 src/libexpr/eval.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 0eb6f406e..d8a36fa02 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -603,8 +603,7 @@ bool isAllowedURI(std::string_view uri, const Strings & allowedUris)
 {
     /* 'uri' should be equal to a prefix, or in a subdirectory of a
        prefix. Thus, the prefix https://github.co does not permit
-       access to https://github.com. Note: this allows 'http://' and
-       'https://' as prefixes for any http/https URI. */
+       access to https://github.com. */
     for (auto & prefix : allowedUris) {
         if (uri == prefix ||
             (uri.size() > prefix.size()

From 1fa958dda1ef0cb37441ef8d1a84faf6d501ac12 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Wed, 6 Dec 2023 14:08:22 +0100
Subject: [PATCH 205/421] isAllowedURI: Format

---
 src/libexpr/eval.cc | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index d8a36fa02..9e541f293 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -605,11 +605,14 @@ bool isAllowedURI(std::string_view uri, const Strings & allowedUris)
        prefix. Thus, the prefix https://github.co does not permit
        access to https://github.com. */
     for (auto & prefix : allowedUris) {
-        if (uri == prefix ||
-            (uri.size() > prefix.size()
-            && prefix.size() > 0
-            && hasPrefix(uri, prefix)
-            && (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/')))
+        if (uri == prefix
+            // Allow access to subdirectories of the prefix.
+            || (uri.size() > prefix.size()
+                && prefix.size() > 0
+                && hasPrefix(uri, prefix)
+                && (
+                    prefix[prefix.size() - 1] == '/'
+                    || uri[prefix.size()] == '/')))
             return true;
     }
 

From 79eb2920bb51c7ec9528a403986e79f04738e2be Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Wed, 6 Dec 2023 14:39:49 +0100
Subject: [PATCH 206/421] Add nix::isASCII*, locale-independent

---
 src/libutil/string.hh        | 17 +++++++++++
 tests/unit/libutil/string.cc | 59 ++++++++++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+)
 create mode 100644 src/libutil/string.hh
 create mode 100644 tests/unit/libutil/string.cc

diff --git a/src/libutil/string.hh b/src/libutil/string.hh
new file mode 100644
index 000000000..16ef75643
--- /dev/null
+++ b/src/libutil/string.hh
@@ -0,0 +1,17 @@
+#pragma once
+
+namespace nix {
+
+    /** Locale-independent version of std::islower(). */
+    inline bool isASCIILower(char c) { return c >= 'a' && c <= 'z'; };
+
+    /** Locale-independent version of std::isupper(). */
+    inline bool isASCIIUpper(char c) { return c >= 'A' && c <= 'Z'; };
+
+    /** Locale-independent version of std::isalpha(). */
+    inline bool isASCIIAlpha(char c) { return isASCIILower(c) || isASCIIUpper(c); };
+
+    /** Locale-independent version of std::isdigit(). */
+    inline bool isASCIIDigit(char c) { return c >= '0' && c <= '9'; };
+
+}
diff --git a/tests/unit/libutil/string.cc b/tests/unit/libutil/string.cc
new file mode 100644
index 000000000..381f2cc15
--- /dev/null
+++ b/tests/unit/libutil/string.cc
@@ -0,0 +1,59 @@
+#include <gtest/gtest.h>
+#include "string.hh"
+
+namespace nix {
+
+TEST(string, isASCIILower) {
+    ASSERT_TRUE(isASCIILower('a'));
+    ASSERT_TRUE(isASCIILower('z'));
+    ASSERT_FALSE(isASCIILower('A'));
+    ASSERT_FALSE(isASCIILower('Z'));
+    ASSERT_FALSE(isASCIILower('0'));
+    ASSERT_FALSE(isASCIILower('9'));
+    ASSERT_FALSE(isASCIILower(' '));
+    ASSERT_FALSE(isASCIILower('\n'));
+    ASSERT_FALSE(isASCIILower('\t'));
+    ASSERT_FALSE(isASCIILower(':'));
+}
+
+TEST(string, isASCIIUpper) {
+    ASSERT_FALSE(isASCIIUpper('a'));
+    ASSERT_FALSE(isASCIIUpper('z'));
+    ASSERT_TRUE(isASCIIUpper('A'));
+    ASSERT_TRUE(isASCIIUpper('Z'));
+    ASSERT_FALSE(isASCIIUpper('0'));
+    ASSERT_FALSE(isASCIIUpper('9'));
+    ASSERT_FALSE(isASCIIUpper(' '));
+    ASSERT_FALSE(isASCIIUpper('\n'));
+    ASSERT_FALSE(isASCIIUpper('\t'));
+    ASSERT_FALSE(isASCIIUpper(':'));
+}
+
+TEST(string, isASCIIAlpha) {
+    ASSERT_TRUE(isASCIIAlpha('a'));
+    ASSERT_TRUE(isASCIIAlpha('z'));
+    ASSERT_TRUE(isASCIIAlpha('A'));
+    ASSERT_TRUE(isASCIIAlpha('Z'));
+    ASSERT_FALSE(isASCIIAlpha('0'));
+    ASSERT_FALSE(isASCIIAlpha('9'));
+    ASSERT_FALSE(isASCIIAlpha(' '));
+    ASSERT_FALSE(isASCIIAlpha('\n'));
+    ASSERT_FALSE(isASCIIAlpha('\t'));
+    ASSERT_FALSE(isASCIIAlpha(':'));
+}
+
+TEST(string, isASCIIDigit) {
+    ASSERT_FALSE(isASCIIDigit('a'));
+    ASSERT_FALSE(isASCIIDigit('z'));
+    ASSERT_FALSE(isASCIIDigit('A'));
+    ASSERT_FALSE(isASCIIDigit('Z'));
+    ASSERT_TRUE(isASCIIDigit('0'));
+    ASSERT_TRUE(isASCIIDigit('1'));
+    ASSERT_TRUE(isASCIIDigit('9'));
+    ASSERT_FALSE(isASCIIDigit(' '));
+    ASSERT_FALSE(isASCIIDigit('\n'));
+    ASSERT_FALSE(isASCIIDigit('\t'));
+    ASSERT_FALSE(isASCIIDigit(':'));
+}
+
+}
\ No newline at end of file

From d3a85b68347071d8d93ec796a38c707483d7b272 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Wed, 6 Dec 2023 15:14:41 +0100
Subject: [PATCH 207/421] isValidSchemeName: Add function

---
 src/libutil/url.cc        | 17 +++++++++++++++++
 src/libutil/url.hh        |  9 +++++++++
 tests/unit/libutil/url.cc | 18 ++++++++++++++++++
 3 files changed, 44 insertions(+)

diff --git a/src/libutil/url.cc b/src/libutil/url.cc
index 57b64d607..f2d5f1782 100644
--- a/src/libutil/url.cc
+++ b/src/libutil/url.cc
@@ -3,6 +3,7 @@
 #include "util.hh"
 #include "split.hh"
 #include "canon-path.hh"
+#include "string.hh"
 
 namespace nix {
 
@@ -183,4 +184,20 @@ std::string fixGitURL(const std::string & url)
     }
 }
 
+// https://www.rfc-editor.org/rfc/rfc3986#section-3.1
+bool isValidSchemeName(std::string_view s)
+{
+    if (s.empty()) return false;
+    if (!isASCIIAlpha(s[0])) return false;
+    for (auto c : s.substr(1)) {
+        if (isASCIIAlpha(c)) continue;
+        if (isASCIIDigit(c)) continue;
+        if (c == '+') continue;
+        if (c == '-') continue;
+        if (c == '.') continue;
+        return false;
+    }
+    return true;
+}
+
 }
diff --git a/src/libutil/url.hh b/src/libutil/url.hh
index 833f54678..24806bbff 100644
--- a/src/libutil/url.hh
+++ b/src/libutil/url.hh
@@ -55,4 +55,13 @@ ParsedUrlScheme parseUrlScheme(std::string_view scheme);
    changes absolute paths into file:// URLs. */
 std::string fixGitURL(const std::string & url);
 
+/**
+ * Whether a string is valid as RFC 3986 scheme name.
+ * Colon `:` is part of the URI; not the scheme name, and therefore rejected.
+ * See https://www.rfc-editor.org/rfc/rfc3986#section-3.1
+ *
+ * Does not check whether the scheme is understood, as that's context-dependent.
+ */
+bool isValidSchemeName(std::string_view scheme);
+
 }
diff --git a/tests/unit/libutil/url.cc b/tests/unit/libutil/url.cc
index a678dad20..09fa4e218 100644
--- a/tests/unit/libutil/url.cc
+++ b/tests/unit/libutil/url.cc
@@ -344,4 +344,22 @@ namespace nix {
         ASSERT_EQ(percentDecode(e), s);
     }
 
+TEST(nix, isValidSchemeName) {
+    ASSERT_TRUE(isValidSchemeName("http"));
+    ASSERT_TRUE(isValidSchemeName("https"));
+    ASSERT_TRUE(isValidSchemeName("file"));
+    ASSERT_TRUE(isValidSchemeName("file+https"));
+    ASSERT_TRUE(isValidSchemeName("fi.le"));
+    ASSERT_TRUE(isValidSchemeName("file-ssh"));
+    ASSERT_TRUE(isValidSchemeName("file+"));
+    ASSERT_TRUE(isValidSchemeName("file."));
+    ASSERT_TRUE(isValidSchemeName("file1"));
+    ASSERT_FALSE(isValidSchemeName("file:"));
+    ASSERT_FALSE(isValidSchemeName("file/"));
+    ASSERT_FALSE(isValidSchemeName("+file"));
+    ASSERT_FALSE(isValidSchemeName(".file"));
+    ASSERT_FALSE(isValidSchemeName("-file"));
+    ASSERT_FALSE(isValidSchemeName("1file"));
+}
+
 }

From a05bc9eb92371af631fc9fb83c3595957fb56943 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Wed, 6 Dec 2023 15:27:29 +0100
Subject: [PATCH 208/421] allowed-uris: Match whole schemes also when scheme is
 not followed by slashes

---
 ...llowed-uris-can-now-match-whole-schemes.md |  7 ++++
 src/libexpr/eval-settings.hh                  |  5 +++
 src/libexpr/eval.cc                           | 17 ++++++++-
 tests/unit/libexpr/eval.cc                    | 35 +++++++++++++++++++
 4 files changed, 63 insertions(+), 1 deletion(-)
 create mode 100644 doc/manual/rl-next/allowed-uris-can-now-match-whole-schemes.md

diff --git a/doc/manual/rl-next/allowed-uris-can-now-match-whole-schemes.md b/doc/manual/rl-next/allowed-uris-can-now-match-whole-schemes.md
new file mode 100644
index 000000000..3cf75a612
--- /dev/null
+++ b/doc/manual/rl-next/allowed-uris-can-now-match-whole-schemes.md
@@ -0,0 +1,7 @@
+---
+synopsis: Option `allowed-uris` can now match whole schemes in URIs without slashes
+prs: 9547
+---
+
+If a scheme, such as `github:` is specified in the `allowed-uris` option, all URIs starting with `github:` are allowed.
+Previously this only worked for schemes whose URIs used the `://` syntax.
diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh
index db2971acb..3009a462c 100644
--- a/src/libexpr/eval-settings.hh
+++ b/src/libexpr/eval-settings.hh
@@ -68,6 +68,11 @@ struct EvalSettings : Config
           evaluation mode. For example, when set to
           `https://github.com/NixOS`, builtin functions such as `fetchGit` are
           allowed to access `https://github.com/NixOS/patchelf.git`.
+
+          Access is granted when
+          - the URI is equal to the prefix,
+          - or the URI is a subpath of the prefix,
+          - or the prefix is a URI scheme ended by a colon `:` and the URI has the same scheme.
         )"};
 
     Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 9e541f293..1552e3e92 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -18,6 +18,7 @@
 #include "memory-input-accessor.hh"
 #include "signals.hh"
 #include "gc-small-vector.hh"
+#include "url.hh"
 
 #include <algorithm>
 #include <chrono>
@@ -599,6 +600,14 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value &
     mkStorePathString(storePath, v);
 }
 
+inline static bool isJustSchemePrefix(std::string_view prefix)
+{
+    return
+        !prefix.empty()
+        && prefix[prefix.size() - 1] == ':'
+        && isValidSchemeName(prefix.substr(0, prefix.size() - 1));
+}
+
 bool isAllowedURI(std::string_view uri, const Strings & allowedUris)
 {
     /* 'uri' should be equal to a prefix, or in a subdirectory of a
@@ -611,8 +620,14 @@ bool isAllowedURI(std::string_view uri, const Strings & allowedUris)
                 && prefix.size() > 0
                 && hasPrefix(uri, prefix)
                 && (
+                    // Allow access to subdirectories of the prefix.
                     prefix[prefix.size() - 1] == '/'
-                    || uri[prefix.size()] == '/')))
+                    || uri[prefix.size()] == '/'
+
+                    // Allow access to whole schemes
+                    || isJustSchemePrefix(prefix)
+                    )
+                ))
             return true;
     }
 
diff --git a/tests/unit/libexpr/eval.cc b/tests/unit/libexpr/eval.cc
index cc5d6bbfa..93d3f658f 100644
--- a/tests/unit/libexpr/eval.cc
+++ b/tests/unit/libexpr/eval.cc
@@ -103,4 +103,39 @@ TEST(nix_isAllowedURI, file_url) {
     ASSERT_FALSE(isAllowedURI("file://", allowed));
 }
 
+TEST(nix_isAllowedURI, github_all) {
+    Strings allowed;
+    allowed.push_back("github:");
+    ASSERT_TRUE(isAllowedURI("github:", allowed));
+    ASSERT_TRUE(isAllowedURI("github:foo/bar", allowed));
+    ASSERT_TRUE(isAllowedURI("github:foo/bar/feat-multi-bar", allowed));
+    ASSERT_TRUE(isAllowedURI("github:foo/bar?ref=refs/heads/feat-multi-bar", allowed));
+    ASSERT_TRUE(isAllowedURI("github://foo/bar", allowed));
+    ASSERT_FALSE(isAllowedURI("https://github:443/foo/bar/archive/master.tar.gz", allowed));
+    ASSERT_FALSE(isAllowedURI("file://github:foo/bar/archive/master.tar.gz", allowed));
+    ASSERT_FALSE(isAllowedURI("file:///github:foo/bar/archive/master.tar.gz", allowed));
+    ASSERT_FALSE(isAllowedURI("github", allowed));
+}
+
+TEST(nix_isAllowedURI, github_org) {
+    Strings allowed;
+    allowed.push_back("github:foo");
+    ASSERT_FALSE(isAllowedURI("github:", allowed));
+    ASSERT_TRUE(isAllowedURI("github:foo/bar", allowed));
+    ASSERT_TRUE(isAllowedURI("github:foo/bar/feat-multi-bar", allowed));
+    ASSERT_TRUE(isAllowedURI("github:foo/bar?ref=refs/heads/feat-multi-bar", allowed));
+    ASSERT_FALSE(isAllowedURI("github://foo/bar", allowed));
+    ASSERT_FALSE(isAllowedURI("https://github:443/foo/bar/archive/master.tar.gz", allowed));
+    ASSERT_FALSE(isAllowedURI("file://github:foo/bar/archive/master.tar.gz", allowed));
+    ASSERT_FALSE(isAllowedURI("file:///github:foo/bar/archive/master.tar.gz", allowed));
+}
+
+TEST(nix_isAllowedURI, non_scheme_colon) {
+    Strings allowed;
+    allowed.push_back("https://foo/bar:");
+    ASSERT_TRUE(isAllowedURI("https://foo/bar:", allowed));
+    ASSERT_TRUE(isAllowedURI("https://foo/bar:/baz", allowed));
+    ASSERT_FALSE(isAllowedURI("https://foo/bar:baz", allowed));
+}
+
 } // namespace nix
\ No newline at end of file

From 89cf53648ca98434a40b0c0cef51fa64f6e0fa37 Mon Sep 17 00:00:00 2001
From: Robert Hensing <roberth@users.noreply.github.com>
Date: Mon, 11 Dec 2023 12:26:31 +0100
Subject: [PATCH 209/421] Contributing branches and reverting (#9577)

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 doc/manual/src/contributing/hacking.md | 42 ++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index 237eff925..4d3d66397 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -282,3 +282,45 @@ See also the [format documentation](https://github.com/haskell/cabal/blob/master
 
 Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`.
 Set `buildUnreleasedNotes = true;` in `flake.nix` to build the release notes on the fly.
+
+## Branches
+
+- [`master`](https://github.com/NixOS/nix/commits/master)
+
+  The main development branch. All changes are approved and merged here.
+  When developing a change, create a branch based on the latest `master`.
+
+  Maintainers try to [keep it in a release-worthy state](#reverting).
+
+- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance)
+
+  These branches are the subject of backports only, and are
+  also [kept](#reverting) in a release-worthy state.
+
+  See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md)
+
+- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release)
+
+  The latest patch release of the latest minor version.
+
+  See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md)
+
+- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport)
+
+  Generally branches created by the backport action.
+
+  See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md)
+
+- [_other_](https://github.com/NixOS/nix/branches/all)
+
+  Branches that do not conform to the above patterns should be feature branches.
+
+## Reverting
+
+If a change turns out to be merged by mistake, or contain a regression, it may be reverted.
+A revert is not a rejection of the contribution, but merely part of an effective development process.
+It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead.
+If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much.
+By embracing reverts as a good part of the development process, everyone wins.
+
+However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try.

From f45d2ee2b7090560fc30a227d638684268af700d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabian=20M=C3=B6ller?= <fabianm88@gmail.com>
Date: Mon, 11 Dec 2023 16:02:09 +0100
Subject: [PATCH 210/421] Fix query parsing for path-like flakes

---
 src/libexpr/flake/flakeref.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc
index 16f45ace7..8b0eb7460 100644
--- a/src/libexpr/flake/flakeref.cc
+++ b/src/libexpr/flake/flakeref.cc
@@ -90,7 +90,7 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
         fragment = percentDecode(url.substr(fragmentStart+1));
     }
     if (pathEnd != std::string::npos && fragmentStart != std::string::npos) {
-        query = decodeQuery(url.substr(pathEnd+1, fragmentStart));
+        query = decodeQuery(url.substr(pathEnd+1, fragmentStart-pathEnd-1));
     }
 
     if (baseDir) {

From 994f1b5c0de44319992ef6b1b106cee3fa400dc4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fabian=20M=C3=B6ller?= <fabianm88@gmail.com>
Date: Mon, 11 Dec 2023 16:05:34 +0100
Subject: [PATCH 211/421] Add test cases for flake urls with fragments

---
 tests/functional/flakes/flakes.sh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh
index ccf1699f9..7506b6b3b 100644
--- a/tests/functional/flakes/flakes.sh
+++ b/tests/functional/flakes/flakes.sh
@@ -193,6 +193,14 @@ nix build -o "$TEST_ROOT/result" flake1
 nix build -o "$TEST_ROOT/result" "$flake1Dir"
 nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir"
 
+# Test explicit packages.default.
+nix build -o "$TEST_ROOT/result" "$flake1Dir#default"
+nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir#default"
+
+# Test explicit packages.default with query.
+nix build -o "$TEST_ROOT/result" "$flake1Dir?ref=HEAD#default"
+nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default"
+
 # Check that store symlinks inside a flake are not interpreted as flakes.
 nix build -o "$flake1Dir/result" "git+file://$flake1Dir"
 nix path-info "$flake1Dir/result"

From 5f30c8acc7e0cad08924cc53e350e811d097fae7 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 10 Dec 2023 18:51:23 -0500
Subject: [PATCH 212/421] Give `Store::queryDerivationOutputMap` and
 `evalStore` argument

Picking up where https://github.com/NixOS/nix/pull/9563 left off.
---
 src/libstore/store-api.cc | 4 ++--
 src/libstore/store-api.hh | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 800df7fa0..7f35e74af 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -547,8 +547,8 @@ std::map<std::string, std::optional<StorePath>> Store::queryPartialDerivationOut
     return outputs;
 }
 
-OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) {
-    auto resp = queryPartialDerivationOutputMap(path);
+OutputPathMap Store::queryDerivationOutputMap(const StorePath & path, Store * evalStore) {
+    auto resp = queryPartialDerivationOutputMap(path, evalStore);
     OutputPathMap result;
     for (auto & [outName, optOutPath] : resp) {
         if (!optOutPath)
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index ada6699d5..13e5a1446 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -369,7 +369,7 @@ public:
      * Query the mapping outputName=>outputPath for the given derivation.
      * Assume every output has a mapping and throw an exception otherwise.
      */
-    OutputPathMap queryDerivationOutputMap(const StorePath & path);
+    OutputPathMap queryDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr);
 
     /**
      * Query the full store path given the hash part of a valid store

From 9f39dda66ce0f92707d4be05d0a90961c78f8bd4 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 10 Dec 2023 21:21:21 -0500
Subject: [PATCH 213/421] Fix building CA derivations with and eval store

I don't love the way this code looks. There are two larger problems:

- eval, build/scratch, destination stores (#5025) should have different
  types to reflect the fact that they are used for different purposes
  and those purposes correspond to different operations. It should be
  impossible to "use the wrong store" in my cases.

- Since drvs can end up in both the eval and build/scratch store, we
  should have some sort of union/layered store (not on the file sytem
  level, just conceptual level) that allows accessing both. This would
  get rid of the ugly "check both" boilerplate in this PR.

Still, it might be better to land this now / soon after minimal cleanup,
so we have a concrete idea of what problem better abstractions are
supposed to solve.
---
 src/libstore/build/derivation-goal.cc | 51 +++++++++++++++++++++------
 src/libstore/misc.cc                  |  9 +++--
 src/libstore/store-api.hh             |  3 +-
 src/nix-build/nix-build.cc            |  6 ++--
 tests/functional/ca/eval-store.sh     | 10 ++++++
 tests/functional/ca/local.mk          |  1 +
 tests/functional/eval-store.sh        | 16 +++++++--
 7 files changed, 76 insertions(+), 20 deletions(-)
 create mode 100644 tests/functional/ca/eval-store.sh

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index d4da374ba..f8728ed4a 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -196,10 +196,19 @@ void DerivationGoal::loadDerivation()
        things being garbage collected while we're busy. */
     worker.evalStore.addTempRoot(drvPath);
 
-    assert(worker.evalStore.isValidPath(drvPath));
+    /* Get the derivation. It is probably in the eval store, but it might be inthe main store:
 
-    /* Get the derivation. */
-    drv = std::make_unique<Derivation>(worker.evalStore.readDerivation(drvPath));
+         - Resolved derivation are resolved against main store realisations, and so must be stored there.
+
+         - Dynamic derivations are built, and so are found in the main store.
+     */
+    for (auto * drvStore : { &worker.evalStore, &worker.store }) {
+        if (drvStore->isValidPath(drvPath)) {
+            drv = std::make_unique<Derivation>(drvStore->readDerivation(drvPath));
+            break;
+        }
+    }
+    assert(drv);
 
     haveDerivation();
 }
@@ -401,11 +410,15 @@ void DerivationGoal::gaveUpOnSubstitution()
     }
 
     /* Copy the input sources from the eval store to the build
-       store. */
+       store.
+
+       Note that some inputs might not be in the eval store because they
+       are (resolved) derivation outputs in a resolved derivation. */
     if (&worker.evalStore != &worker.store) {
         RealisedPath::Set inputSrcs;
         for (auto & i : drv->inputSrcs)
-            inputSrcs.insert(i);
+            if (worker.evalStore.isValidPath(i))
+                inputSrcs.insert(i);
         copyClosure(worker.evalStore, worker.store, inputSrcs);
     }
 
@@ -453,7 +466,7 @@ void DerivationGoal::repairClosure()
     std::map<StorePath, StorePath> outputsToDrv;
     for (auto & i : inputClosure)
         if (i.isDerivation()) {
-            auto depOutputs = worker.store.queryPartialDerivationOutputMap(i);
+            auto depOutputs = worker.store.queryPartialDerivationOutputMap(i, &worker.evalStore);
             for (auto & j : depOutputs)
                 if (j.second)
                     outputsToDrv.insert_or_assign(*j.second, i);
@@ -604,7 +617,13 @@ void DerivationGoal::inputsRealised()
                     return *outPath;
                 }
                 else {
-                    auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath);
+                    auto outMap = [&]{
+                        for (auto * drvStore : { &worker.evalStore, &worker.store })
+                            if (drvStore->isValidPath(depDrvPath))
+                                return worker.store.queryDerivationOutputMap(depDrvPath, drvStore);
+                        assert(false);
+                    }();
+
                     auto outMapPath = outMap.find(outputName);
                     if (outMapPath == outMap.end()) {
                         throw Error(
@@ -1085,8 +1104,12 @@ void DerivationGoal::resolvedFinished()
                 auto newRealisation = realisation;
                 newRealisation.id = DrvOutput { initialOutput->outputHash, outputName };
                 newRealisation.signatures.clear();
-                if (!drv->type().isFixed())
-                    newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath);
+                if (!drv->type().isFixed()) {
+                    auto & drvStore = worker.evalStore.isValidPath(drvPath)
+                        ? worker.evalStore
+                        : worker.store;
+                    newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore);
+                }
                 signRealisation(newRealisation);
                 worker.store.registerDrvOutput(newRealisation);
             }
@@ -1379,7 +1402,10 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri
             res.insert_or_assign(name, output.path(worker.store, drv->name, name));
         return res;
     } else {
-        return worker.store.queryPartialDerivationOutputMap(drvPath);
+        for (auto * drvStore : { &worker.evalStore, &worker.store })
+            if (drvStore->isValidPath(drvPath))
+                return worker.store.queryPartialDerivationOutputMap(drvPath, drvStore);
+        assert(false);
     }
 }
 
@@ -1392,7 +1418,10 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
             res.insert_or_assign(name, *output.second);
         return res;
     } else {
-        return worker.store.queryDerivationOutputMap(drvPath);
+        for (auto * drvStore : { &worker.evalStore, &worker.store })
+            if (drvStore->isValidPath(drvPath))
+                return worker.store.queryDerivationOutputMap(drvPath, drvStore);
+        assert(false);
     }
 }
 
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 9f63fbbb5..cc8ad3d02 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -331,8 +331,11 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
 std::map<DrvOutput, StorePath> drvOutputReferences(
     Store & store,
     const Derivation & drv,
-    const StorePath & outputPath)
+    const StorePath & outputPath,
+    Store * evalStore_)
 {
+    auto & evalStore = evalStore_ ? *evalStore_ : store;
+
     std::set<Realisation> inputRealisations;
 
     std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumRealisations;
@@ -340,7 +343,7 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
     accumRealisations = [&](const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
         if (!inputNode.value.empty()) {
             auto outputHashes =
-                staticOutputHashes(store, store.readDerivation(inputDrv));
+                staticOutputHashes(evalStore, evalStore.readDerivation(inputDrv));
             for (const auto & outputName : inputNode.value) {
                 auto outputHash = get(outputHashes, outputName);
                 if (!outputHash)
@@ -362,7 +365,7 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
                 SingleDerivedPath next = SingleDerivedPath::Built { d, outputName };
                 accumRealisations(
                     // TODO deep resolutions for dynamic derivations, issue #8947, would go here.
-                    resolveDerivedPath(store, next),
+                    resolveDerivedPath(store, next, evalStore_),
                     childNode);
             }
         }
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 13e5a1446..2c883ce97 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -943,6 +943,7 @@ const ContentAddress * getDerivationCA(const BasicDerivation & drv);
 std::map<DrvOutput, StorePath> drvOutputReferences(
     Store & store,
     const Derivation & drv,
-    const StorePath & outputPath);
+    const StorePath & outputPath,
+    Store * evalStore = nullptr);
 
 }
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 01da028d8..8e9be14c1 100644
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -462,7 +462,7 @@ static void main_nix_build(int argc, char * * argv)
         if (dryRun) return;
 
         if (shellDrv) {
-            auto shellDrvOutputs = store->queryPartialDerivationOutputMap(shellDrv.value());
+            auto shellDrvOutputs = store->queryPartialDerivationOutputMap(shellDrv.value(), &*evalStore);
             shell = store->printStorePath(shellDrvOutputs.at("out").value()) + "/bin/bash";
         }
 
@@ -515,7 +515,7 @@ static void main_nix_build(int argc, char * * argv)
             std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputClosure;
 
             accumInputClosure = [&](const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
-                auto outputs = evalStore->queryPartialDerivationOutputMap(inputDrv);
+                auto outputs = store->queryPartialDerivationOutputMap(inputDrv, &*evalStore);
                 for (auto & i : inputNode.value) {
                     auto o = outputs.at(i);
                     store->computeFSClosure(*o, inputs);
@@ -653,7 +653,7 @@ static void main_nix_build(int argc, char * * argv)
             if (counter)
                 drvPrefix += fmt("-%d", counter + 1);
 
-            auto builtOutputs = evalStore->queryPartialDerivationOutputMap(drvPath);
+            auto builtOutputs = store->queryPartialDerivationOutputMap(drvPath, &*evalStore);
 
             auto maybeOutputPath = builtOutputs.at(outputName);
             assert(maybeOutputPath);
diff --git a/tests/functional/ca/eval-store.sh b/tests/functional/ca/eval-store.sh
new file mode 100644
index 000000000..9cc499606
--- /dev/null
+++ b/tests/functional/ca/eval-store.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# Ensure that garbage collection works properly with ca derivations
+
+source common.sh
+
+export NIX_TESTS_CA_BY_DEFAULT=1
+
+cd ..
+source eval-store.sh
diff --git a/tests/functional/ca/local.mk b/tests/functional/ca/local.mk
index fd87b8d1f..4f86b268f 100644
--- a/tests/functional/ca/local.mk
+++ b/tests/functional/ca/local.mk
@@ -5,6 +5,7 @@ ca-tests := \
   $(d)/concurrent-builds.sh \
   $(d)/derivation-json.sh \
   $(d)/duplicate-realisation-in-closure.sh \
+  $(d)/eval-store.sh \
   $(d)/gc.sh \
   $(d)/import-derivation.sh \
   $(d)/new-build-cmd.sh \
diff --git a/tests/functional/eval-store.sh b/tests/functional/eval-store.sh
index 8fc859730..ec99fd953 100644
--- a/tests/functional/eval-store.sh
+++ b/tests/functional/eval-store.sh
@@ -11,7 +11,16 @@ rm -rf "$eval_store"
 
 nix build -f dependencies.nix --eval-store "$eval_store" -o "$TEST_ROOT/result"
 [[ -e $TEST_ROOT/result/foobar ]]
-(! ls $NIX_STORE_DIR/*.drv)
+if [[ ! -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then
+    # Resolved CA derivations are written to store for building
+    #
+    # TODO when we something more systematic
+    # (https://github.com/NixOS/nix/issues/5025) that distinguishes
+    # between scratch storage for building and the final destination
+    # store, we'll be able to make this unconditional again -- resolved
+    # derivations should only appear in the scratch store.
+    (! ls $NIX_STORE_DIR/*.drv)
+fi
 ls $eval_store/nix/store/*.drv
 
 clearStore
@@ -26,5 +35,8 @@ rm -rf "$eval_store"
 
 nix-build dependencies.nix --eval-store "$eval_store" -o "$TEST_ROOT/result"
 [[ -e $TEST_ROOT/result/foobar ]]
-(! ls $NIX_STORE_DIR/*.drv)
+if [[ ! -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then
+    # See above
+    (! ls $NIX_STORE_DIR/*.drv)
+fi
 ls $eval_store/nix/store/*.drv

From 0b81557e2cf30cebb916f82f192f04df38c810d7 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 6 Dec 2023 20:14:14 -0500
Subject: [PATCH 214/421] flake.nix: Put some list items on their own line

These things are about to become longer
---
 flake.nix | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/flake.nix b/flake.nix
index 99480183a..973bb55af 100644
--- a/flake.nix
+++ b/flake.nix
@@ -36,8 +36,10 @@
       systems = linuxSystems ++ darwinSystems;
 
       crossSystems = [
-        "armv6l-linux" "armv7l-linux"
-        "x86_64-freebsd13" "x86_64-netbsd"
+        "armv6l-linux"
+        "armv7l-linux"
+        "x86_64-freebsd13"
+        "x86_64-netbsd"
       ];
 
       stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ];
@@ -575,8 +577,25 @@
         # to https://nixos.org/nix/install. It downloads the binary
         # tarball for the user's system and calls the second half of the
         # installation script.
-        installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ];
-        installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"];
+        installerScript = installScriptFor [
+          # Native
+          "x86_64-linux"
+          "i686-linux"
+          "aarch64-linux"
+          "x86_64-darwin"
+          "aarch64-darwin"
+          # Cross
+          "armv6l-linux"
+          "armv7l-linux"
+        ];
+        installerScriptForGHA = installScriptFor [
+          # Native
+          "x86_64-linux"
+          "x86_64-darwin"
+          # Cross
+          "armv6l-linux"
+          "armv7l-linux"
+        ];
 
         # docker image with Nix inside
         dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage);

From f60c2e8a5acabf5fb554f77014904bb0d0c91604 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 6 Dec 2023 20:28:11 -0500
Subject: [PATCH 215/421] flake.nix: `installScriptFor` take tarballs not
 strings

Trying to look up keys in multiple places is not nice, better for the
caller to be explicit.
---
 flake.nix | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/flake.nix b/flake.nix
index 973bb55af..185140092 100644
--- a/flake.nix
+++ b/flake.nix
@@ -256,7 +256,7 @@
           ];
       };
 
-      installScriptFor = systems:
+      installScriptFor = tarballs:
         with nixpkgsFor.x86_64-linux.native;
         runCommand "installer-script"
           { buildInputs = [ nix ];
@@ -277,14 +277,14 @@
 
             substitute ${./scripts/install.in} $out/install \
               ${pkgs.lib.concatMapStrings
-                (system: let
-                    tarball = if builtins.elem system crossSystems then self.hydraJobs.binaryTarballCross.x86_64-linux.${system} else self.hydraJobs.binaryTarball.${system};
+                (tarball: let
+                    inherit (tarball.stdenv.hostPlatform) system;
                   in '' \
                   --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \
                   --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \
                   ''
                 )
-                systems
+                tarballs
               } --replace '@nixVersion@' ${version}
 
             echo "file installer $out/install" >> $out/nix-support/hydra-build-products
@@ -341,7 +341,7 @@
           installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix cacert ]; };
         in
 
-        buildPackages.runCommand "nix-binary-tarball-${version}"
+        pkgs.runCommand "nix-binary-tarball-${version}"
           { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
             meta.description = "Distribution-independent Nix bootstrap binaries for ${pkgs.system}";
           }
@@ -579,22 +579,22 @@
         # installation script.
         installerScript = installScriptFor [
           # Native
-          "x86_64-linux"
-          "i686-linux"
-          "aarch64-linux"
-          "x86_64-darwin"
-          "aarch64-darwin"
+          self.hydraJobs.binaryTarball."x86_64-linux"
+          self.hydraJobs.binaryTarball."i686-linux"
+          self.hydraJobs.binaryTarball."aarch64-linux"
+          self.hydraJobs.binaryTarball."x86_64-darwin"
+          self.hydraJobs.binaryTarball."aarch64-darwin"
           # Cross
-          "armv6l-linux"
-          "armv7l-linux"
+          self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-linux"
+          self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-linux"
         ];
         installerScriptForGHA = installScriptFor [
           # Native
-          "x86_64-linux"
-          "x86_64-darwin"
+          self.hydraJobs.binaryTarball."x86_64-linux"
+          self.hydraJobs.binaryTarball."x86_64-darwin"
           # Cross
-          "armv6l-linux"
-          "armv7l-linux"
+          self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-linux"
+          self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-linux"
         ];
 
         # docker image with Nix inside

From 78492cfde73d57ca01c73d77a23440754c9e7ee4 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 2 Sep 2023 14:36:25 -0400
Subject: [PATCH 216/421] flake.nix: Use `config` not `system` for cross so we
 can be a bit more precise

---
 flake.nix                     | 20 ++++++++++----------
 maintainers/upload-release.pl |  4 ++--
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/flake.nix b/flake.nix
index 185140092..2f6668b81 100644
--- a/flake.nix
+++ b/flake.nix
@@ -36,10 +36,10 @@
       systems = linuxSystems ++ darwinSystems;
 
       crossSystems = [
-        "armv6l-linux"
-        "armv7l-linux"
-        "x86_64-freebsd13"
-        "x86_64-netbsd"
+        "armv6l-unknown-linux-gnueabihf"
+        "armv7l-unknown-linux-gnueabihf"
+        "x86_64-unknown-freebsd13"
+        "x86_64-unknown-netbsd"
       ];
 
       stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ];
@@ -116,8 +116,8 @@
               inherit system;
             };
             crossSystem = if crossSystem == null then null else {
-              system = crossSystem;
-            } // lib.optionalAttrs (crossSystem == "x86_64-freebsd13") {
+              config = crossSystem;
+            } // lib.optionalAttrs (crossSystem == "x86_64-unknown-freebsd13") {
               useLLVM = true;
             };
             overlays = [
@@ -585,16 +585,16 @@
           self.hydraJobs.binaryTarball."x86_64-darwin"
           self.hydraJobs.binaryTarball."aarch64-darwin"
           # Cross
-          self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-linux"
-          self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-linux"
+          self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf"
+          self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
         ];
         installerScriptForGHA = installScriptFor [
           # Native
           self.hydraJobs.binaryTarball."x86_64-linux"
           self.hydraJobs.binaryTarball."x86_64-darwin"
           # Cross
-          self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-linux"
-          self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-linux"
+          self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf"
+          self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
         ];
 
         # docker image with Nix inside
diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl
index ebc536f12..4e2c379f0 100755
--- a/maintainers/upload-release.pl
+++ b/maintainers/upload-release.pl
@@ -154,8 +154,8 @@ downloadFile("binaryTarball.x86_64-linux", "1");
 downloadFile("binaryTarball.aarch64-linux", "1");
 downloadFile("binaryTarball.x86_64-darwin", "1");
 downloadFile("binaryTarball.aarch64-darwin", "1");
-downloadFile("binaryTarballCross.x86_64-linux.armv6l-linux", "1");
-downloadFile("binaryTarballCross.x86_64-linux.armv7l-linux", "1");
+downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1");
+downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1");
 downloadFile("installerScript", "1");
 
 # Upload docker images to dockerhub.

From 46b98a40a7c5488a99525bc780b7f7bba0131545 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 2 Sep 2023 14:38:33 -0400
Subject: [PATCH 217/421] flake.nix: Make changes so a MinGW dev shell would
 work

---
 flake.nix | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/flake.nix b/flake.nix
index 2f6668b81..f0dd9a3e3 100644
--- a/flake.nix
+++ b/flake.nix
@@ -210,7 +210,7 @@
 
         buildDeps =
           [ curl
-            bzip2 xz brotli editline
+            bzip2 xz brotli
             openssl sqlite
             libarchive
             (pkgs.libgit2.overrideAttrs (attrs: {
@@ -219,10 +219,13 @@
               cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"];
             }))
             boost
-            lowdown-nix
             libsodium
           ]
-          ++ lib.optionals stdenv.isLinux [libseccomp]
+          ++ lib.optionals (!stdenv.hostPlatform.isWindows) [
+            editline
+            lowdown-nix
+          ]
+          ++ lib.optional stdenv.isLinux libseccomp
           ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid;
 
         checkDeps = [
@@ -510,7 +513,7 @@
               stdenv = currentStdenv;
             };
 
-            meta.platforms = lib.platforms.unix;
+            meta.platforms = lib.platforms.unix ++ lib.platforms.windows;
             meta.mainProgram = "nix";
           });
 

From b892161e314d976e7692ffcf487e1aa042165745 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 11 Dec 2023 12:26:42 -0500
Subject: [PATCH 218/421] flake.nix: Make a MinGW dev shell

This requires a `shellCrossSystems` for now, since Nix doesn't actually
build on Windows. This can be dropped once it does.
---
 flake.nix | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/flake.nix b/flake.nix
index f0dd9a3e3..bbdfc38e9 100644
--- a/flake.nix
+++ b/flake.nix
@@ -42,6 +42,13 @@
         "x86_64-unknown-netbsd"
       ];
 
+      # Nix doesn't yet build on this platform, so we put it in a
+      # separate list. We just use this for `devShells` and
+      # `nixpkgsFor`, which this depends on.
+      shellCrossSystems = crossSystems ++ [
+        "x86_64-w64-mingw32"
+      ];
+
       stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ];
 
       forAllSystems = lib.genAttrs systems;
@@ -129,7 +136,7 @@
         in {
           inherit stdenvs native;
           static = native.pkgsStatic;
-          cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv");
+          cross = lib.genAttrs shellCrossSystems (crossSystem: make-pkgs crossSystem "stdenv");
         });
 
       commonDeps =
@@ -808,7 +815,7 @@
           in
             (makeShells "native" nixpkgsFor.${system}.native) //
             (makeShells "static" nixpkgsFor.${system}.static) //
-            (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) //
+            (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) //
             {
               default = self.devShells.${system}.native-stdenvPackages;
             }

From 589fb105f311af65230d374cbbddf7173c7ad103 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <theophane.hufschmitt@tweag.io>
Date: Tue, 12 Dec 2023 16:05:32 +0100
Subject: [PATCH 219/421] Fix the VM tests

Work around https://github.com/NixOS/nixpkgs/issues/271146 until we can
depend on a Nixpkgs version containing https://github.com/NixOS/nixpkgs/pull/271423
---
 flake.nix               | 4 ++++
 tests/nixos/default.nix | 1 +
 2 files changed, 5 insertions(+)

diff --git a/flake.nix b/flake.nix
index bbdfc38e9..ada52c05d 100644
--- a/flake.nix
+++ b/flake.nix
@@ -7,6 +7,10 @@
   # Also, do not grab arbitrary further staging commits. This PR was
   # carefully made to be based on release-23.05 and just contain
   # rebuild-causing changes to packages that Nix actually uses.
+  #
+  # Once this is updated to something containing
+  # https://github.com/NixOS/nixpkgs/pull/271423, don't forget
+  # to remove the `nix.checkAllErrors = false;` line in the tests.
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/staging-23.05";
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix
index 4459aa664..2645cac8e 100644
--- a/tests/nixos/default.nix
+++ b/tests/nixos/default.nix
@@ -10,6 +10,7 @@ let
     hostPkgs = nixpkgsFor.${system}.native;
     defaults = {
       nixpkgs.pkgs = nixpkgsFor.${system}.native;
+      nix.checkAllErrors = false;
     };
     _module.args.nixpkgs = nixpkgs;
   };

From 2e451a663eff96b89360cfd3c0d5eaa60ca46181 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Tue, 12 Dec 2023 17:22:54 +0100
Subject: [PATCH 220/421] schemeRegex -> schemeNameRegex

Scheme could be understood to include the typical `:` separator.
---
 src/libutil/url-parts.hh | 2 +-
 src/libutil/url.cc       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh
index 5c5a30dc2..07bc8d0cd 100644
--- a/src/libutil/url-parts.hh
+++ b/src/libutil/url-parts.hh
@@ -8,7 +8,7 @@ namespace nix {
 
 // URI stuff.
 const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
-const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)";
+const static std::string schemeNameRegex = "(?:[a-z][a-z0-9+.-]*)";
 const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+(?:%\\w+)?";
 const static std::string ipv6AddressRegex = "(?:\\[" + ipv6AddressSegmentRegex + "\\]|" + ipv6AddressSegmentRegex + ")";
 const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
diff --git a/src/libutil/url.cc b/src/libutil/url.cc
index f2d5f1782..e9acd67d0 100644
--- a/src/libutil/url.cc
+++ b/src/libutil/url.cc
@@ -14,7 +14,7 @@ std::regex revRegex(revRegexS, std::regex::ECMAScript);
 ParsedURL parseURL(const std::string & url)
 {
     static std::regex uriRegex(
-        "((" + schemeRegex + "):"
+        "((" + schemeNameRegex + "):"
         + "(?:(?://(" + authorityRegex + ")(" + absPathRegex + "))|(/?" + pathRegex + ")))"
         + "(?:\\?(" + queryRegex + "))?"
         + "(?:#(" + queryRegex + "))?",

From 4eaeda6604e2f8977728f14415fe92350d047970 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Tue, 12 Dec 2023 17:43:54 +0100
Subject: [PATCH 221/421] isValidSchemeName: Use regex

As requested by Eelco Dolstra. I think it used to be simpler.
---
 src/libutil/url.cc        | 15 +++------------
 tests/unit/libutil/url.cc |  5 +++++
 2 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/src/libutil/url.cc b/src/libutil/url.cc
index e9acd67d0..152c06d8e 100644
--- a/src/libutil/url.cc
+++ b/src/libutil/url.cc
@@ -3,7 +3,6 @@
 #include "util.hh"
 #include "split.hh"
 #include "canon-path.hh"
-#include "string.hh"
 
 namespace nix {
 
@@ -187,17 +186,9 @@ std::string fixGitURL(const std::string & url)
 // https://www.rfc-editor.org/rfc/rfc3986#section-3.1
 bool isValidSchemeName(std::string_view s)
 {
-    if (s.empty()) return false;
-    if (!isASCIIAlpha(s[0])) return false;
-    for (auto c : s.substr(1)) {
-        if (isASCIIAlpha(c)) continue;
-        if (isASCIIDigit(c)) continue;
-        if (c == '+') continue;
-        if (c == '-') continue;
-        if (c == '.') continue;
-        return false;
-    }
-    return true;
+    static std::regex regex(schemeNameRegex, std::regex::ECMAScript);
+
+    return std::regex_match(s.begin(), s.end(), regex, std::regex_constants::match_default);
 }
 
 }
diff --git a/tests/unit/libutil/url.cc b/tests/unit/libutil/url.cc
index 09fa4e218..7d08f467e 100644
--- a/tests/unit/libutil/url.cc
+++ b/tests/unit/libutil/url.cc
@@ -360,6 +360,11 @@ TEST(nix, isValidSchemeName) {
     ASSERT_FALSE(isValidSchemeName(".file"));
     ASSERT_FALSE(isValidSchemeName("-file"));
     ASSERT_FALSE(isValidSchemeName("1file"));
+    // regex ok?
+    ASSERT_FALSE(isValidSchemeName("\nhttp"));
+    ASSERT_FALSE(isValidSchemeName("\nhttp\n"));
+    ASSERT_FALSE(isValidSchemeName("http\n"));
+    ASSERT_FALSE(isValidSchemeName("http "));
 }
 
 }

From 0b87ba50c08d83384e11a8e6db1e2f97fba4b61c Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Tue, 12 Dec 2023 17:44:53 +0100
Subject: [PATCH 222/421] Revert "Add nix::isASCII*, locale-independent"

This reverts commit 79eb2920bb51c7ec9528a403986e79f04738e2be.

Not used at this time.
---
 src/libutil/string.hh        | 17 -----------
 tests/unit/libutil/string.cc | 59 ------------------------------------
 2 files changed, 76 deletions(-)
 delete mode 100644 src/libutil/string.hh
 delete mode 100644 tests/unit/libutil/string.cc

diff --git a/src/libutil/string.hh b/src/libutil/string.hh
deleted file mode 100644
index 16ef75643..000000000
--- a/src/libutil/string.hh
+++ /dev/null
@@ -1,17 +0,0 @@
-#pragma once
-
-namespace nix {
-
-    /** Locale-independent version of std::islower(). */
-    inline bool isASCIILower(char c) { return c >= 'a' && c <= 'z'; };
-
-    /** Locale-independent version of std::isupper(). */
-    inline bool isASCIIUpper(char c) { return c >= 'A' && c <= 'Z'; };
-
-    /** Locale-independent version of std::isalpha(). */
-    inline bool isASCIIAlpha(char c) { return isASCIILower(c) || isASCIIUpper(c); };
-
-    /** Locale-independent version of std::isdigit(). */
-    inline bool isASCIIDigit(char c) { return c >= '0' && c <= '9'; };
-
-}
diff --git a/tests/unit/libutil/string.cc b/tests/unit/libutil/string.cc
deleted file mode 100644
index 381f2cc15..000000000
--- a/tests/unit/libutil/string.cc
+++ /dev/null
@@ -1,59 +0,0 @@
-#include <gtest/gtest.h>
-#include "string.hh"
-
-namespace nix {
-
-TEST(string, isASCIILower) {
-    ASSERT_TRUE(isASCIILower('a'));
-    ASSERT_TRUE(isASCIILower('z'));
-    ASSERT_FALSE(isASCIILower('A'));
-    ASSERT_FALSE(isASCIILower('Z'));
-    ASSERT_FALSE(isASCIILower('0'));
-    ASSERT_FALSE(isASCIILower('9'));
-    ASSERT_FALSE(isASCIILower(' '));
-    ASSERT_FALSE(isASCIILower('\n'));
-    ASSERT_FALSE(isASCIILower('\t'));
-    ASSERT_FALSE(isASCIILower(':'));
-}
-
-TEST(string, isASCIIUpper) {
-    ASSERT_FALSE(isASCIIUpper('a'));
-    ASSERT_FALSE(isASCIIUpper('z'));
-    ASSERT_TRUE(isASCIIUpper('A'));
-    ASSERT_TRUE(isASCIIUpper('Z'));
-    ASSERT_FALSE(isASCIIUpper('0'));
-    ASSERT_FALSE(isASCIIUpper('9'));
-    ASSERT_FALSE(isASCIIUpper(' '));
-    ASSERT_FALSE(isASCIIUpper('\n'));
-    ASSERT_FALSE(isASCIIUpper('\t'));
-    ASSERT_FALSE(isASCIIUpper(':'));
-}
-
-TEST(string, isASCIIAlpha) {
-    ASSERT_TRUE(isASCIIAlpha('a'));
-    ASSERT_TRUE(isASCIIAlpha('z'));
-    ASSERT_TRUE(isASCIIAlpha('A'));
-    ASSERT_TRUE(isASCIIAlpha('Z'));
-    ASSERT_FALSE(isASCIIAlpha('0'));
-    ASSERT_FALSE(isASCIIAlpha('9'));
-    ASSERT_FALSE(isASCIIAlpha(' '));
-    ASSERT_FALSE(isASCIIAlpha('\n'));
-    ASSERT_FALSE(isASCIIAlpha('\t'));
-    ASSERT_FALSE(isASCIIAlpha(':'));
-}
-
-TEST(string, isASCIIDigit) {
-    ASSERT_FALSE(isASCIIDigit('a'));
-    ASSERT_FALSE(isASCIIDigit('z'));
-    ASSERT_FALSE(isASCIIDigit('A'));
-    ASSERT_FALSE(isASCIIDigit('Z'));
-    ASSERT_TRUE(isASCIIDigit('0'));
-    ASSERT_TRUE(isASCIIDigit('1'));
-    ASSERT_TRUE(isASCIIDigit('9'));
-    ASSERT_FALSE(isASCIIDigit(' '));
-    ASSERT_FALSE(isASCIIDigit('\n'));
-    ASSERT_FALSE(isASCIIDigit('\t'));
-    ASSERT_FALSE(isASCIIDigit(':'));
-}
-
-}
\ No newline at end of file

From 04f454f2a0e1bfb4fc0368872f215cb690df11bc Mon Sep 17 00:00:00 2001
From: SharzyL <me@sharzy.in>
Date: Wed, 13 Dec 2023 10:30:28 +0800
Subject: [PATCH 223/421] fix: nix copy ssh-ng:// not respecting
 --substitute-on-destination

---
 src/libstore/remote-store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index cc26c2a94..dd6347468 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -225,7 +225,7 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
         conn->to << WorkerProto::Op::QueryValidPaths;
         WorkerProto::write(*this, *conn, paths);
         if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) {
-            conn->to << (settings.buildersUseSubstitutes ? 1 : 0);
+            conn->to << maybeSubstitute;
         }
         conn.processStderr();
         return WorkerProto::Serialise<StorePathSet>::read(*this, *conn);

From cc3913e4584beb19e7af00572db119d2638333d5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 13 Dec 2023 13:27:23 +0100
Subject: [PATCH 224/421] Remove unused variable

---
 src/libexpr/eval.hh | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index f3f6d35b9..f452dcb9f 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -335,11 +335,6 @@ private:
 
     std::map<std::string, std::optional<std::string>> searchPathResolved;
 
-    /**
-     * Cache used by checkSourcePath().
-     */
-    std::unordered_map<Path, SourcePath> resolvedPaths;
-
     /**
      * Cache used by prim_match().
      */

From 103ca0bde5d4f32745d4c3aee534cf4aa0a69a9d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 13 Dec 2023 13:27:29 +0100
Subject: [PATCH 225/421] Improve SourcePath display

---
 src/libfetchers/input-accessor.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh
index d5ac238b1..f385e6231 100644
--- a/src/libfetchers/input-accessor.hh
+++ b/src/libfetchers/input-accessor.hh
@@ -130,7 +130,7 @@ struct SourcePath
     { return accessor->getPhysicalPath(path); }
 
     std::string to_string() const
-    { return path.abs(); }
+    { return accessor->showPath(path); }
 
     /**
      * Append a `CanonPath` to this path.

From faa4cae9aed4e9f8c40ed8c6fe00bd0216c3b0ea Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 13 Dec 2023 13:27:39 +0100
Subject: [PATCH 226/421] LibExprTest: Ignore $NIX_PATH

Otherwise a broken $NIX_PATH can cause the test suite to fail.
---
 tests/unit/libexpr-support/tests/libexpr.hh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/unit/libexpr-support/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh
index 968431446..d720cedde 100644
--- a/tests/unit/libexpr-support/tests/libexpr.hh
+++ b/tests/unit/libexpr-support/tests/libexpr.hh
@@ -8,6 +8,7 @@
 #include "nixexpr.hh"
 #include "eval.hh"
 #include "eval-inline.hh"
+#include "eval-settings.hh"
 #include "store-api.hh"
 
 #include "tests/libstore.hh"
@@ -18,6 +19,7 @@ namespace nix {
             static void SetUpTestSuite() {
                 LibStoreTest::SetUpTestSuite();
                 initGC();
+                evalSettings.nixPath = {};
             }
 
         protected:

From 19ec1c9fd4d4bf6e941b046b8549ba2a1a690937 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 13 Dec 2023 15:15:30 +0100
Subject: [PATCH 227/421] Improve the unsafeGetAttrPos test

We can use corepkgsFS->addFile() now to create a "real" position.
---
 tests/unit/libexpr/primops.cc | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/tests/unit/libexpr/primops.cc b/tests/unit/libexpr/primops.cc
index 7485fa0d0..384d9924b 100644
--- a/tests/unit/libexpr/primops.cc
+++ b/tests/unit/libexpr/primops.cc
@@ -1,6 +1,8 @@
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 
+#include "memory-input-accessor.hh"
+
 #include "tests/libexpr.hh"
 
 namespace nix {
@@ -148,10 +150,25 @@ namespace nix {
     }
 
     TEST_F(PrimOpTest, unsafeGetAttrPos) {
-        // The `y` attribute is at position
-        const char* expr = "builtins.unsafeGetAttrPos \"y\" { y = \"x\"; }";
+        state.corepkgsFS->addFile(CanonPath("foo.nix"), "{ y = \"x\"; }");
+
+        auto expr = "builtins.unsafeGetAttrPos \"y\" (import <nix/foo.nix>)";
         auto v = eval(expr);
-        ASSERT_THAT(v, IsNull());
+        ASSERT_THAT(v, IsAttrsOfSize(3));
+
+        auto file = v.attrs->find(createSymbol("file"));
+        ASSERT_NE(file, nullptr);
+        ASSERT_THAT(*file->value, IsString());
+        auto s = baseNameOf(file->value->string_view());
+        ASSERT_EQ(s, "foo.nix");
+
+        auto line = v.attrs->find(createSymbol("line"));
+        ASSERT_NE(line, nullptr);
+        ASSERT_THAT(*line->value, IsIntEq(1));
+
+        auto column = v.attrs->find(createSymbol("column"));
+        ASSERT_NE(column, nullptr);
+        ASSERT_THAT(*column->value, IsIntEq(3));
     }
 
     TEST_F(PrimOpTest, hasAttr) {

From e76df8781417dad9ab4f0a6c3b28917e35f204bf Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 13 Dec 2023 11:26:14 -0500
Subject: [PATCH 228/421] Test `nix copy --substitute-on-destination`

It works with both `ssh://` and `ssh-ng://` now since #9600 (and
`ssh-ng:// didn't work before that).

Also, by making the two tests share code, we nudge ourselves towards
making sure there is feature parity.
---
 tests/functional/nix-copy-ssh-common.sh | 70 +++++++++++++++++++++++++
 tests/functional/nix-copy-ssh-ng.sh     | 16 +++---
 tests/functional/nix-copy-ssh.sh        | 19 +------
 3 files changed, 77 insertions(+), 28 deletions(-)
 create mode 100644 tests/functional/nix-copy-ssh-common.sh

diff --git a/tests/functional/nix-copy-ssh-common.sh b/tests/functional/nix-copy-ssh-common.sh
new file mode 100644
index 000000000..cc8314ff7
--- /dev/null
+++ b/tests/functional/nix-copy-ssh-common.sh
@@ -0,0 +1,70 @@
+proto=$1
+shift
+(( $# == 0 ))
+
+clearStore
+clearCache
+
+mkdir -p $TEST_ROOT/stores
+
+# Create path to copy back and forth
+outPath=$(nix-build --no-out-link dependencies.nix)
+
+storeQueryParam="store=${NIX_STORE_DIR}"
+
+realQueryParam () {
+    echo "real=$1$NIX_STORE_DIR"
+}
+
+remoteRoot="$TEST_ROOT/stores/$proto"
+
+clearRemoteStore () {
+    chmod -R u+w "$remoteRoot" || true
+    rm -rf "$remoteRoot"
+}
+
+clearRemoteStore
+
+remoteStore="${proto}://localhost?${storeQueryParam}&remote-store=${remoteRoot}%3f${storeQueryParam}%26$(realQueryParam "$remoteRoot")"
+
+# Copy to store
+
+args=()
+if [[ "$proto" == "ssh-ng" ]]; then
+    # TODO investigate discrepancy
+    args+=(--no-check-sigs)
+fi
+
+[ ! -f ${remoteRoot}${outPath}/foobar ]
+nix copy "${args[@]}" --to "$remoteStore" $outPath
+[ -f ${remoteRoot}${outPath}/foobar ]
+
+# Copy back from store
+
+clearStore
+
+[ ! -f $outPath/foobar ]
+nix copy --no-check-sigs --from "$remoteStore" $outPath
+[ -f $outPath/foobar ]
+
+# Check --substitute-on-destination, avoid corrupted store
+
+clearRemoteStore
+
+corruptedRoot=$TEST_ROOT/stores/corrupted
+corruptedStore="${corruptedRoot}?${storeQueryParam}&$(realQueryParam "$corruptedRoot")"
+
+# Copy it to the corrupted store
+nix copy --no-check-sigs "$outPath" --to "$corruptedStore"
+
+# Corrupt it in there
+corruptPath="${corruptedRoot}${outPath}"
+chmod +w "$corruptPath"
+echo "not supposed to be here" > "$corruptPath/foobarbaz"
+chmod -w "$corruptPath"
+
+# Copy from the corrupted store with the regular store as a
+# substituter. It must use the substituter not the source store in
+# order to avoid errors.
+NIX_CONFIG=$(echo -e "substituters = local\nrequire-sigs = false") \
+    nix copy --no-check-sigs --from "$corruptedStore" --to "$remoteStore" --substitute-on-destination "$outPath"
diff --git a/tests/functional/nix-copy-ssh-ng.sh b/tests/functional/nix-copy-ssh-ng.sh
index 463b5e0c4..62e99cd24 100644
--- a/tests/functional/nix-copy-ssh-ng.sh
+++ b/tests/functional/nix-copy-ssh-ng.sh
@@ -1,18 +1,14 @@
 source common.sh
 
-clearStore
-clearCache
+source nix-copy-ssh-common.sh "ssh-ng"
 
-remoteRoot=$TEST_ROOT/store2
-chmod -R u+w "$remoteRoot" || true
-rm -rf "$remoteRoot"
+clearStore
+clearRemoteStore
 
 outPath=$(nix-build --no-out-link dependencies.nix)
 
-nix store info --store "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR"
+nix store info --store "$remoteStore"
 
 # Regression test for https://github.com/NixOS/nix/issues/6253
-nix copy --to "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath --no-check-sigs &
-nix copy --to "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath --no-check-sigs
-
-[ -f $remoteRoot$outPath/foobar ]
+nix copy --to "$remoteStore" $outPath --no-check-sigs &
+nix copy --to "$remoteStore" $outPath --no-check-sigs
diff --git a/tests/functional/nix-copy-ssh.sh b/tests/functional/nix-copy-ssh.sh
index eb801548d..12e8346bc 100644
--- a/tests/functional/nix-copy-ssh.sh
+++ b/tests/functional/nix-copy-ssh.sh
@@ -1,20 +1,3 @@
 source common.sh
 
-clearStore
-clearCache
-
-remoteRoot=$TEST_ROOT/store2
-chmod -R u+w "$remoteRoot" || true
-rm -rf "$remoteRoot"
-
-outPath=$(nix-build --no-out-link dependencies.nix)
-
-nix copy --to "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath
-
-[ -f $remoteRoot$outPath/foobar ]
-
-clearStore
-
-nix copy --no-check-sigs --from "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath
-
-[ -f $outPath/foobar ]
+source nix-copy-ssh-common.sh "ssh"

From 19573f1b05b7d3ccfd07c9c351396494d488ab2d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 13 Dec 2023 15:33:15 -0500
Subject: [PATCH 229/421] Restore comment

---
 scripts/binary-tarball.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/binary-tarball.nix b/scripts/binary-tarball.nix
index 32e811c94..104189b0c 100644
--- a/scripts/binary-tarball.nix
+++ b/scripts/binary-tarball.nix
@@ -14,6 +14,7 @@ let
   inherit (nix) version;
 
   env = {
+    #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
     meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
   };
 

From f10f0f1b50228e09ad587a7c550df586061e4514 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 13 Dec 2023 20:41:20 +0000
Subject: [PATCH 230/421] Move `lowdown.nix` to `misc/`

---
 flake.nix                       | 2 +-
 lowdown.nix => misc/lowdown.nix | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename lowdown.nix => misc/lowdown.nix (100%)

diff --git a/flake.nix b/flake.nix
index 0cdd2b41f..c7ff7eb64 100644
--- a/flake.nix
+++ b/flake.nix
@@ -167,7 +167,7 @@
             '';
           };
 
-          lowdown-nix = final.callPackage ./lowdown.nix {
+          lowdown-nix = final.callPackage ./misc/lowdown.nix {
             inherit lowdown-src stdenv;
           };
 
diff --git a/lowdown.nix b/misc/lowdown.nix
similarity index 100%
rename from lowdown.nix
rename to misc/lowdown.nix

From bf5804d46a0d0aa5eb40107b6eaeec4e95bbd4a2 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 13 Dec 2023 20:41:41 +0000
Subject: [PATCH 231/421] flake.nix: Delete uneeded `attrs0` binding

---
 package.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.nix b/package.nix
index 7c1ba3130..8fc4be328 100644
--- a/package.nix
+++ b/package.nix
@@ -92,7 +92,7 @@
 , __forDefaults ? {
     canRunInstalled = doBuild && stdenv.buildPlatform.canExecute stdenv.hostPlatform;
   }
-} @ attrs0:
+}:
 
 let
   version = lib.fileContents ./.version + versionSuffix;

From 28f2f3136d19ef7de4c6acd9678aef72e80d4fb8 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 13 Dec 2023 20:47:36 +0000
Subject: [PATCH 232/421] Delete stray `install_name_tool` call

---
 package.nix | 1 -
 1 file changed, 1 deletion(-)

diff --git a/package.nix b/package.nix
index 8fc4be328..0b5b512c7 100644
--- a/package.nix
+++ b/package.nix
@@ -320,7 +320,6 @@ in {
       -change ${boost}/lib/libboost_context.dylib \
       $out/lib/libboost_context.dylib \
       $out/lib/libnixutil.dylib
-      install_name_tool
     ''
   ) + lib.optionalString enableInternalAPIDocs ''
     mkdir -p ''${!outputDoc}/nix-support

From 2d24875fe4aa7f31d15acfc29b9aa5c45109f99d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 13 Dec 2023 20:49:31 +0000
Subject: [PATCH 233/421] package.nix: Avoid `${..}` for conditional strings

Using `+` is Nixpkgs standard ideom for this, and helps avoid needless
rebuilds somewhat.
---
 package.nix | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/package.nix b/package.nix
index 0b5b512c7..0b2ff43b0 100644
--- a/package.nix
+++ b/package.nix
@@ -254,25 +254,25 @@ in {
 
   disallowedReferences = [ boost ];
 
-  preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) ''
-    # Copy libboost_context so we don't get all of Boost in our closure.
-    # https://github.com/NixOS/nixpkgs/issues/45462
-    mkdir -p $out/lib
-    cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib
-    rm -f $out/lib/*.a
-    ${lib.optionalString stdenv.hostPlatform.isLinux ''
+  preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) (
+    ''
+      # Copy libboost_context so we don't get all of Boost in our closure.
+      # https://github.com/NixOS/nixpkgs/issues/45462
+      mkdir -p $out/lib
+      cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib
+      rm -f $out/lib/*.a
+    '' + lib.optionalString stdenv.hostPlatform.isLinux ''
       chmod u+w $out/lib/*.so.*
       patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
-    ''}
-    ${lib.optionalString stdenv.hostPlatform.isDarwin ''
+    '' + lib.optionalString stdenv.hostPlatform.isDarwin ''
       for LIB in $out/lib/*.dylib; do
         chmod u+w $LIB
         install_name_tool -id $LIB $LIB
         install_name_tool -delete_rpath ${boost}/lib/ $LIB || true
       done
       install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
-    ''}
-  '';
+    ''
+  );
 
   configureFlags = [
     "--sysconfdir=/etc"

From 7b29b44d8e62f686aa9fbfafe53be959cdba03cb Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 13 Dec 2023 16:22:35 -0500
Subject: [PATCH 234/421] Remove custom lowdown

This was last upgraded in 788008385ef5bf7edb799977525d6f73f02c76bc, but
the version in Nixpkgs is a now a lot newer. I think the custom was
added to get ahead of Nixpkgs before, and so now that we are in fact
behind, it is no longer needed.
---
 flake.lock       | 17 -----------------
 flake.nix        | 11 +----------
 misc/lowdown.nix | 22 ----------------------
 3 files changed, 1 insertion(+), 49 deletions(-)
 delete mode 100644 misc/lowdown.nix

diff --git a/flake.lock b/flake.lock
index 3cb9e72c9..db1a72c14 100644
--- a/flake.lock
+++ b/flake.lock
@@ -32,22 +32,6 @@
         "type": "github"
       }
     },
-    "lowdown-src": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1633514407,
-        "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=",
-        "owner": "kristapsdz",
-        "repo": "lowdown",
-        "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8",
-        "type": "github"
-      },
-      "original": {
-        "owner": "kristapsdz",
-        "repo": "lowdown",
-        "type": "github"
-      }
-    },
     "nixpkgs": {
       "locked": {
         "lastModified": 1701355166,
@@ -84,7 +68,6 @@
       "inputs": {
         "flake-compat": "flake-compat",
         "libgit2": "libgit2",
-        "lowdown-src": "lowdown-src",
         "nixpkgs": "nixpkgs",
         "nixpkgs-regression": "nixpkgs-regression"
       }
diff --git a/flake.nix b/flake.nix
index c7ff7eb64..eb3846564 100644
--- a/flake.nix
+++ b/flake.nix
@@ -13,11 +13,10 @@
   # to remove the `nix.checkAllErrors = false;` line in the tests.
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/staging-23.05";
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
-  inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
   inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; };
 
-  outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, libgit2, ... }:
+  outputs = { self, nixpkgs, nixpkgs-regression, libgit2, ... }:
 
     let
       inherit (nixpkgs) lib;
@@ -140,9 +139,6 @@
         {
           nixStable = prev.nix;
 
-          # Forward from the previous stage as we don’t want it to pick the lowdown override
-          inherit (prev) nixUnstable;
-
           default-busybox-sandbox-shell = final.busybox.override {
             useMusl = true;
             enableStatic = true;
@@ -167,10 +163,6 @@
             '';
           };
 
-          lowdown-nix = final.callPackage ./misc/lowdown.nix {
-            inherit lowdown-src stdenv;
-          };
-
           libgit2-nix = final.libgit2.overrideAttrs (attrs: {
             src = libgit2;
             version = libgit2.lastModifiedDate;
@@ -208,7 +200,6 @@
               officialRelease = false;
               boehmgc = final.boehmgc-nix;
               libgit2 = final.libgit2-nix;
-              lowdown = final.lowdown-nix;
               busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
               changelog-d = final.changelog-d-nix;
             } // {
diff --git a/misc/lowdown.nix b/misc/lowdown.nix
deleted file mode 100644
index 5f469fad5..000000000
--- a/misc/lowdown.nix
+++ /dev/null
@@ -1,22 +0,0 @@
-{ lib
-, stdenv
-, which
-, lowdown-src
-}:
-
-stdenv.mkDerivation rec {
-  name = "lowdown-0.9.0";
-
-  src = lowdown-src;
-
-  outputs = [ "out" "bin" "dev" ];
-
-  nativeBuildInputs = [ which ];
-
-  configurePhase = ''
-      ${lib.optionalString (stdenv.isDarwin && stdenv.isAarch64) "echo \"HAVE_SANDBOX_INIT=false\" > configure.local"}
-      ./configure \
-        PREFIX=${placeholder "dev"} \
-        BINDIR=${placeholder "bin"}/bin
-  '';
-}

From 1e3d8118401d80da54fb64641e606042c3499e4d Mon Sep 17 00:00:00 2001
From: Ramses <ramses@well-founded.dev>
Date: Wed, 13 Dec 2023 22:37:17 +0100
Subject: [PATCH 235/421] worker protocol: serialise cgroup stats in
 `BuildResult` (#9598)

By doing so, they get reported when building through the daemon via either `unix://` or `ssh-ng://`.
---
 doc/manual/rl-next/cgroup-stats.md            |   8 +++
 src/libstore/worker-protocol.cc               |  34 ++++++++++
 src/libstore/worker-protocol.hh               |   6 +-
 .../worker-protocol/build-result-1.37.bin     | Bin 0 -> 808 bytes
 tests/unit/libstore/worker-protocol.cc        |  61 ++++++++++++++++--
 5 files changed, 101 insertions(+), 8 deletions(-)
 create mode 100644 doc/manual/rl-next/cgroup-stats.md
 create mode 100644 tests/unit/libstore/data/worker-protocol/build-result-1.37.bin

diff --git a/doc/manual/rl-next/cgroup-stats.md b/doc/manual/rl-next/cgroup-stats.md
new file mode 100644
index 000000000..00853a0f8
--- /dev/null
+++ b/doc/manual/rl-next/cgroup-stats.md
@@ -0,0 +1,8 @@
+---
+synopsis: Include cgroup stats when building through the daemon
+prs: 9598
+---
+
+Nix now also reports cgroup statistics when building through the nix daemon and when doing remote builds using ssh-ng,
+if both sides of the connection are this version of Nix or newer. 
+
diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc
index 2a379e75e..a50259d24 100644
--- a/src/libstore/worker-protocol.cc
+++ b/src/libstore/worker-protocol.cc
@@ -7,6 +7,7 @@
 #include "archive.hh"
 #include "path-info.hh"
 
+#include <chrono>
 #include <nlohmann/json.hpp>
 
 namespace nix {
@@ -47,6 +48,31 @@ void WorkerProto::Serialise<std::optional<TrustedFlag>>::write(const StoreDirCon
 }
 
 
+std::optional<std::chrono::microseconds> WorkerProto::Serialise<std::optional<std::chrono::microseconds>>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn)
+{
+    auto tag = readNum<uint8_t>(conn.from);
+    switch (tag) {
+        case 0:
+            return std::nullopt;
+        case 1:
+            return std::optional<std::chrono::microseconds>{std::chrono::microseconds(readNum<int64_t>(conn.from))};
+        default:
+            throw Error("Invalid optional tag from remote");
+    }
+}
+
+void WorkerProto::Serialise<std::optional<std::chrono::microseconds>>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const std::optional<std::chrono::microseconds> & optDuration)
+{
+    if (!optDuration.has_value()) {
+        conn.to << uint8_t{0};
+    } else {
+        conn.to
+            << uint8_t{1}
+            << optDuration.value().count();
+    }
+}
+
+
 DerivedPath WorkerProto::Serialise<DerivedPath>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn)
 {
     auto s = readString(conn.from);
@@ -110,6 +136,10 @@ BuildResult WorkerProto::Serialise<BuildResult>::read(const StoreDirConfig & sto
             >> res.startTime
             >> res.stopTime;
     }
+    if (GET_PROTOCOL_MINOR(conn.version) >= 37) {
+        res.cpuUser = WorkerProto::Serialise<std::optional<std::chrono::microseconds>>::read(store, conn);
+        res.cpuSystem = WorkerProto::Serialise<std::optional<std::chrono::microseconds>>::read(store, conn);
+    }
     if (GET_PROTOCOL_MINOR(conn.version) >= 28) {
         auto builtOutputs = WorkerProto::Serialise<DrvOutputs>::read(store, conn);
         for (auto && [output, realisation] : builtOutputs)
@@ -132,6 +162,10 @@ void WorkerProto::Serialise<BuildResult>::write(const StoreDirConfig & store, Wo
             << res.startTime
             << res.stopTime;
     }
+    if (GET_PROTOCOL_MINOR(conn.version) >= 37) {
+        WorkerProto::write(store, conn, res.cpuUser);
+        WorkerProto::write(store, conn, res.cpuSystem);
+    }
     if (GET_PROTOCOL_MINOR(conn.version) >= 28) {
         DrvOutputs builtOutputs;
         for (auto & [output, realisation] : res.builtOutputs)
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index c26914289..91d277b77 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -1,6 +1,8 @@
 #pragma once
 ///@file
 
+#include <chrono>
+
 #include "common-protocol.hh"
 
 namespace nix {
@@ -9,7 +11,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION (1 << 8 | 36)
+#define PROTOCOL_VERSION (1 << 8 | 37)
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
@@ -214,6 +216,8 @@ template<>
 DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo);
 template<>
 DECLARE_WORKER_SERIALISER(std::optional<TrustedFlag>);
+template<>
+DECLARE_WORKER_SERIALISER(std::optional<std::chrono::microseconds>);
 
 template<typename T>
 DECLARE_WORKER_SERIALISER(std::vector<T>);
diff --git a/tests/unit/libstore/data/worker-protocol/build-result-1.37.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.37.bin
new file mode 100644
index 0000000000000000000000000000000000000000..7d6e43fff2f593c9669897f1594ef2526293b866
GIT binary patch
literal 808
zcmc(b!A=4(5QbNYCw&D7HXbqAZP|9&SMZ?mYCOy`Q??0dfh{3~@J_yn?_}3497ssi
zkk~`NPG{zy$$yh{=Qh&1p+SP-rryS%zu_*nozv~b{8i*2l1Kg&hyFwTsm?J^pZ&Jx
z7(XWuZG7Ec;XHLnni_a6OQ{Pv(Gvn*a8V&-GN)9gN@`4z#zM+q67p$E1#+ya@Ky@P
zI?Ja*37y|pu=-Z~h`Kw5v>=OQ{VT!TG~kW14J&v15i`h2cEQPP#N67yfUkq@EZeAh
nE0*W@7*-7pjhR{S>lKBa-ro0@_Cq`OPkw~Szw@JOIPv2Pj)JK^

literal 0
HcmV?d00001

diff --git a/tests/unit/libstore/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc
index 91f804f0c..2b2e559a9 100644
--- a/tests/unit/libstore/worker-protocol.cc
+++ b/tests/unit/libstore/worker-protocol.cc
@@ -280,13 +280,60 @@ VERSIONED_CHARACTERIZATION_TEST(
                 },
                 .startTime = 30,
                 .stopTime = 50,
-#if 0
-                // These fields are not yet serialized.
-                // FIXME Include in next version of protocol or document
-                // why they are skipped.
-                .cpuUser = std::chrono::milliseconds(500s),
-                .cpuSystem = std::chrono::milliseconds(604s),
-#endif
+            },
+        };
+        t;
+    }))
+
+VERSIONED_CHARACTERIZATION_TEST(
+    WorkerProtoTest,
+    buildResult_1_37,
+    "build-result-1.37",
+    1 << 8 | 37,
+    ({
+        using namespace std::literals::chrono_literals;
+        std::tuple<BuildResult, BuildResult, BuildResult> t {
+            BuildResult {
+                .status = BuildResult::OutputRejected,
+                .errorMsg = "no idea why",
+            },
+            BuildResult {
+                .status = BuildResult::NotDeterministic,
+                .errorMsg = "no idea why",
+                .timesBuilt = 3,
+                .isNonDeterministic = true,
+                .startTime = 30,
+                .stopTime = 50,
+            },
+            BuildResult {
+                .status = BuildResult::Built,
+                .timesBuilt = 1,
+                .builtOutputs = {
+                    {
+                        "foo",
+                        {
+                            .id = DrvOutput {
+                                .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
+                                .outputName = "foo",
+                            },
+                            .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
+                        },
+                    },
+                    {
+                        "bar",
+                        {
+                            .id = DrvOutput {
+                                .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
+                                .outputName = "bar",
+                            },
+                            .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" },
+                        },
+                    },
+                },
+                .startTime = 30,
+                .stopTime = 50,
+                .cpuUser = std::chrono::microseconds(500s),
+                .cpuSystem = std::chrono::microseconds(604s),
             },
         };
         t;

From 6ed803737c587a0cc9026093c941c1d1172fa5dc Mon Sep 17 00:00:00 2001
From: Rebecca Turner <rbt@sent.as>
Date: Wed, 13 Dec 2023 14:02:52 -0800
Subject: [PATCH 236/421] Use `--with-boost` on macOS

`configureFlags` only included `--with-boost` on Linux, which makes
local builds as outlined in `doc/manual/src/contributing/hacking.md`
fail when performed on macOS.
---
 package.nix | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/package.nix b/package.nix
index 0b2ff43b0..24395b484 100644
--- a/package.nix
+++ b/package.nix
@@ -284,8 +284,9 @@ in {
   ] ++ lib.optionals installUnitTests [
     "--with-check-bin-dir=${builtins.placeholder "check"}/bin"
     "--with-check-lib-dir=${builtins.placeholder "check"}/lib"
-  ] ++ lib.optionals (doBuild && stdenv.isLinux) [
+  ] ++ lib.optionals (doBuild) [
     "--with-boost=${boost}/lib"
+  ] ++ lib.optionals (doBuild && stdenv.isLinux) [
     "--with-sandbox-shell=${busybox-sandbox-shell}/bin/busybox"
   ] ++ lib.optional (doBuild && stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux"))
        "LDFLAGS=-fuse-ld=gold"

From e13fc0bbdb1e1eefeb33ff4d18310958041b1ad5 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 8 Dec 2023 19:52:08 -0500
Subject: [PATCH 237/421] Fix `sys/xattr.h` check

I wrote the `configure.ac` wrong, and so we just got no builds
supporting ACLs.

Also, it needs to be more precise because Darwin puts other stuff in
that same header, evidently.
---
 configure.ac                          | 3 ++-
 src/libstore/posix-fs-canonicalise.cc | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/configure.ac b/configure.ac
index f9ad3c840..a949f9df2 100644
--- a/configure.ac
+++ b/configure.ac
@@ -289,7 +289,8 @@ esac
 AC_SUBST(HAVE_SECCOMP, [$have_seccomp])
 
 # Optional dependencies for better normalizing file system data
-AC_CHECK_HEADERS[sys/xattr.h]
+AC_CHECK_HEADERS([sys/xattr.h])
+AC_CHECK_FUNCS([llistxattr lremovexattr])
 
 # Look for aws-cpp-sdk-s3.
 AC_LANG_PUSH(C++)
diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc
index f38fa8369..5edda0157 100644
--- a/src/libstore/posix-fs-canonicalise.cc
+++ b/src/libstore/posix-fs-canonicalise.cc
@@ -78,7 +78,7 @@ static void canonicalisePathMetaData_(
     if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
         throw Error("file '%1%' has an unsupported type", path);
 
-#ifdef HAVE_SYS_XATTR_H
+#if HAVE_SYS_XATTR_H && HAVE_LLISTXATTR && HAVE_LREMOVEXATTR
     /* Remove extended attributes / ACLs. */
     ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0);
 

From 8d39c0c19638eb0cd07c1d0af89320e33f9c02d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20Luis=20Lafuente?= <jl@lafuente.dev>
Date: Thu, 14 Dec 2023 23:14:59 +0100
Subject: [PATCH 238/421] Fix clang devshell

Issue introduced in https://github.com/NixOS/nix/pull/9535
---
 package.nix | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/package.nix b/package.nix
index 24395b484..6ea5bf9c9 100644
--- a/package.nix
+++ b/package.nix
@@ -5,6 +5,7 @@
 , autoreconfHook
 , aws-sdk-cpp
 , boehmgc
+, buildPackages
 , nlohmann_json
 , bison
 , boost
@@ -207,6 +208,9 @@ in {
     # changelog
     ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d
     ++ lib.optional enableInternalAPIDocs doxygen
+
+    ++ lib.optional stdenv.cc.isClang buildPackages.bear
+    ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) buildPackages.clang-tools
   ;
 
   buildInputs = lib.optionals doBuild [

From bcbdb09ccf7ca007d3c2046177356fbfe7b72304 Mon Sep 17 00:00:00 2001
From: Matthew Bauer <mjbauer95@gmail.com>
Date: Tue, 29 Sep 2020 15:33:47 -0400
Subject: [PATCH 239/421] Add eval-system option

`eval-system` option overrides just the value of `builtins.currentSystem`.
This is more useful than overriding `system` since you can build these
derivations on remote builders which can work on the given system.

Co-authored-by: John Ericson <John.Ericson@Obsidian.Systems>
Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 src/libexpr/eval-settings.cc  |  6 ++++++
 src/libexpr/eval-settings.hh  | 20 ++++++++++++++++++++
 src/libexpr/primops.cc        | 11 +++++++----
 src/libstore/globals.hh       |  6 +++++-
 tests/unit/libexpr/primops.cc |  3 ++-
 5 files changed, 40 insertions(+), 6 deletions(-)

diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc
index 444a7d7d6..2ccbe327f 100644
--- a/src/libexpr/eval-settings.cc
+++ b/src/libexpr/eval-settings.cc
@@ -89,6 +89,12 @@ std::string EvalSettings::resolvePseudoUrl(std::string_view url)
         return std::string(url);
 }
 
+const std::string & EvalSettings::getCurrentSystem()
+{
+    const auto & evalSystem = currentSystem.get();
+    return evalSystem != "" ? evalSystem : settings.thisSystem.get();
+}
+
 EvalSettings evalSettings;
 
 static GlobalConfig::Register rEvalSettings(&evalSettings);
diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh
index 3009a462c..ad187ca01 100644
--- a/src/libexpr/eval-settings.hh
+++ b/src/libexpr/eval-settings.hh
@@ -27,6 +27,26 @@ struct EvalSettings : Config
           [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath).
         )"};
 
+    Setting<std::string> currentSystem{
+        this, "", "eval-system",
+        R"(
+          This option defines
+          [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem)
+          in the Nix language if it is set as a non-empty string.
+          Otherwise, if it is defined as the empty string (the default), the value of the
+          [`system` ](#conf-system)
+          configuration setting is used instead.
+
+          Unlike `system`, this setting does not change what kind of derivations can be built locally.
+          This is useful for evaluating Nix code on one system to produce derivations to be built on another type of system.
+        )"};
+
+    /**
+     * Implements the `eval-system` vs `system` defaulting logic
+     * described for `eval-system`.
+     */
+    const std::string & getCurrentSystem();
+
     Setting<bool> restrictEval{
         this, false, "restrict-eval",
         R"(
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 89d5492da..d78a28c73 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -4383,13 +4383,16 @@ void EvalState::createBaseEnv()
         .impureOnly = true,
     });
 
-    if (!evalSettings.pureEval) {
-        v.mkString(settings.thisSystem.get());
-    }
+    if (!evalSettings.pureEval)
+        v.mkString(evalSettings.getCurrentSystem());
     addConstant("__currentSystem", v, {
         .type = nString,
         .doc = R"(
-          The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-system).
+          The value of the
+          [`eval-system`](@docroot@/command-ref/conf-file.md#conf-eval-system)
+          or else
+          [`system`](@docroot@/command-ref/conf-file.md#conf-system)
+          configuration option.
 
           It can be used to set the `system` attribute for [`builtins.derivation`](@docroot@/language/derivations.md) such that the resulting derivation can be built on the same system that evaluates the Nix expression:
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index df977e294..e28615cdc 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -214,7 +214,11 @@ public:
           In general, you do not have to modify this setting.
           While you can force Nix to run a Darwin-specific `builder` executable on a Linux machine, the result would obviously be wrong.
 
-          This value is available in the Nix language as [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem).
+          This value is available in the Nix language as
+          [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem)
+          if the
+          [`eval-system`](#conf-eval-system)
+          configuration option is set as the empty string.
         )"};
 
     Setting<time_t> maxSilentTime{
diff --git a/tests/unit/libexpr/primops.cc b/tests/unit/libexpr/primops.cc
index 384d9924b..31b1b49ae 100644
--- a/tests/unit/libexpr/primops.cc
+++ b/tests/unit/libexpr/primops.cc
@@ -1,6 +1,7 @@
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 
+#include "eval-settings.hh"
 #include "memory-input-accessor.hh"
 
 #include "tests/libexpr.hh"
@@ -631,7 +632,7 @@ namespace nix {
 
     TEST_F(PrimOpTest, currentSystem) {
         auto v = eval("builtins.currentSystem");
-        ASSERT_THAT(v, IsStringEq(settings.thisSystem.get()));
+        ASSERT_THAT(v, IsStringEq(evalSettings.getCurrentSystem()));
     }
 
     TEST_F(PrimOpTest, derivation) {

From 70f50cbb2aa35f1ad1e38c9c73a5f8267baac17d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 14 Dec 2023 19:47:10 -0500
Subject: [PATCH 240/421] Functional Test for `builtins.storeDir` and
 `builtins.currentSystem`

These were under-tested. This tests the status quo and especially
previous commit of this PR better.
---
 tests/functional/impure-eval.sh | 35 +++++++++++++++++++++++++++++++++
 tests/functional/local.mk       |  1 +
 2 files changed, 36 insertions(+)
 create mode 100644 tests/functional/impure-eval.sh

diff --git a/tests/functional/impure-eval.sh b/tests/functional/impure-eval.sh
new file mode 100644
index 000000000..6c72f01d7
--- /dev/null
+++ b/tests/functional/impure-eval.sh
@@ -0,0 +1,35 @@
+source common.sh
+
+export REMOTE_STORE="dummy://"
+
+simpleTest () {
+    local expr=$1; shift
+    local result=$1; shift
+    # rest, extra args
+
+    [[ "$(nix eval --impure --raw "$@" --expr "$expr")" == "$result" ]]
+}
+
+# `builtins.storeDir`
+
+## Store dir follows `store` store setting
+simpleTest 'builtins.storeDir' '/foo' --store "$REMOTE_STORE?store=/foo"
+simpleTest 'builtins.storeDir' '/bar' --store "$REMOTE_STORE?store=/bar"
+
+# `builtins.currentSystem`
+
+## `system` alone affects by default
+simpleTest 'builtins.currentSystem' 'foo' --system 'foo'
+simpleTest 'builtins.currentSystem' 'bar' --system 'bar'
+
+## `system` affects if `eval-system` is an empty string
+simpleTest 'builtins.currentSystem' 'foo' --system 'foo' --eval-system ''
+simpleTest 'builtins.currentSystem' 'bar' --system 'bar' --eval-system ''
+
+## `eval-system` alone affects
+simpleTest 'builtins.currentSystem' 'foo' --eval-system 'foo'
+simpleTest 'builtins.currentSystem' 'bar' --eval-system 'bar'
+
+## `eval-system` overrides `system`
+simpleTest 'builtins.currentSystem' 'bar' --system 'foo' --eval-system 'bar'
+simpleTest 'builtins.currentSystem' 'baz' --system 'foo' --eval-system 'baz'
diff --git a/tests/functional/local.mk b/tests/functional/local.mk
index 10b399d75..192e275e3 100644
--- a/tests/functional/local.mk
+++ b/tests/functional/local.mk
@@ -71,6 +71,7 @@ nix_tests = \
   build-remote-trustless-should-fail-0.sh \
   build-remote-with-mounted-ssh-ng.sh \
   nar-access.sh \
+  impure-eval.sh \
   pure-eval.sh \
   eval.sh \
   repl.sh \

From 228e995cde0f059e4edebdfc8f46d3389d2dc135 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 14 Dec 2023 19:53:59 -0500
Subject: [PATCH 241/421] Add release not for `eval-system`

---
 doc/manual/rl-next/eval-system.md | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/manual/rl-next/eval-system.md

diff --git a/doc/manual/rl-next/eval-system.md b/doc/manual/rl-next/eval-system.md
new file mode 100644
index 000000000..a4696a56c
--- /dev/null
+++ b/doc/manual/rl-next/eval-system.md
@@ -0,0 +1,12 @@
+---
+synopsis: Add new `eval-system` setting
+prs: 4093
+---
+
+Add a new `eval-system` option.
+Unlike `system`, it just overrides the value of `builtins.currentSystem`.
+This is more useful than overriding `system`, because you can build these derivations on remote builders which can work on the given system.
+In contrast, `system` also effects scheduling which will cause Nix to build those derivations locally even if that doesn't make sense.
+
+`eval-system` only takes effect if it is non-empty.
+If empty (the default) `system` is used as before, so there is no breakage.

From 66d37b73383e40f0362b82a0e29c60d2913d689a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20Luis=20Lafuente?= <jl@lafuente.dev>
Date: Fri, 15 Dec 2023 12:41:38 +0100
Subject: [PATCH 242/421] Move clang dev deps to the nix devshell override

---
 flake.nix   | 5 ++++-
 package.nix | 4 ----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/flake.nix b/flake.nix
index eb3846564..8c4436729 100644
--- a/flake.nix
+++ b/flake.nix
@@ -395,7 +395,7 @@
           stdenvs)));
 
       devShells = let
-        makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; }).overrideAttrs (_: {
+        makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; }).overrideAttrs (attrs: {
           installFlags = "sysconfdir=$(out)/etc";
           shellHook = ''
             PATH=$prefix/bin:$PATH
@@ -405,6 +405,9 @@
             # Make bash completion work.
             XDG_DATA_DIRS+=:$out/share
           '';
+          nativeBuildInputs = attrs.nativeBuildInputs or []
+            ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear
+            ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools;
         });
         in
         forAllSystems (system:
diff --git a/package.nix b/package.nix
index 6ea5bf9c9..24395b484 100644
--- a/package.nix
+++ b/package.nix
@@ -5,7 +5,6 @@
 , autoreconfHook
 , aws-sdk-cpp
 , boehmgc
-, buildPackages
 , nlohmann_json
 , bison
 , boost
@@ -208,9 +207,6 @@ in {
     # changelog
     ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d
     ++ lib.optional enableInternalAPIDocs doxygen
-
-    ++ lib.optional stdenv.cc.isClang buildPackages.bear
-    ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) buildPackages.clang-tools
   ;
 
   buildInputs = lib.optionals doBuild [

From 5cb98095ba2c3de83d32c1729da7b9f6cfb1aeff Mon Sep 17 00:00:00 2001
From: Rebecca Turner <rbt@sent.as>
Date: Fri, 15 Dec 2023 23:56:17 -0800
Subject: [PATCH 243/421] Remove some blank lines from stack traces
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This keeps hint messages, source location information, and source code
snippets grouped together, while making stack traces shorter (so that
more stack frames can be viewed on the same terminal).

Before:

    error:
           … while evaluating the attribute 'body'

             at /Users/wiggles/nix/tests/functional/lang/eval-fail-assert.nix:4:3:

                3|
                4|   body = x "x";
                 |   ^
                5| }

           … from call site

             at /Users/wiggles/nix/tests/functional/lang/eval-fail-assert.nix:4:10:

                3|
                4|   body = x "x";
                 |          ^
                5| }

           … while calling 'x'

             at /Users/wiggles/nix/tests/functional/lang/eval-fail-assert.nix:2:7:

                1| let {
                2|   x = arg: assert arg == "y"; 123;
                 |       ^
                3|

           error: assertion '(arg == "y")' failed

           at /Users/wiggles/nix/tests/functional/lang/eval-fail-assert.nix:2:12:

                1| let {
                2|   x = arg: assert arg == "y"; 123;
                 |            ^
                3|

After:

    error:
           … while evaluating the attribute 'body'
             at /Users/wiggles/nix/tests/functional/lang/eval-fail-assert.nix:4:3:
                3|
                4|   body = x "x";
                 |   ^
                5| }

           … from call site
             at /Users/wiggles/nix/tests/functional/lang/eval-fail-assert.nix:4:10:
                3|
                4|   body = x "x";
                 |          ^
                5| }

           … while calling 'x'
             at /Users/wiggles/nix/tests/functional/lang/eval-fail-assert.nix:2:7:
                1| let {
                2|   x = arg: assert arg == "y"; 123;
                 |       ^
                3|

           error: assertion '(arg == "y")' failed
           at /Users/wiggles/nix/tests/functional/lang/eval-fail-assert.nix:2:12:
                1| let {
                2|   x = arg: assert arg == "y"; 123;
                 |            ^
                3|
---
 src/libutil/error.cc                                 |  3 +--
 tests/functional/lang/eval-fail-abort.err.exp        |  2 --
 ...il-addDrvOutputDependencies-empty-context.err.exp |  2 --
 ...dDrvOutputDependencies-multi-elem-context.err.exp |  2 --
 ...dDrvOutputDependencies-wrong-element-kind.err.exp |  2 --
 tests/functional/lang/eval-fail-assert.err.exp       |  8 --------
 .../functional/lang/eval-fail-attr-name-type.err.exp |  4 ----
 .../eval-fail-bad-string-interpolation-1.err.exp     |  2 --
 .../eval-fail-bad-string-interpolation-3.err.exp     |  2 --
 .../eval-fail-bad-string-interpolation-4.err.exp     |  2 --
 tests/functional/lang/eval-fail-blackhole.err.exp    |  4 ----
 tests/functional/lang/eval-fail-call-primop.err.exp  |  2 --
 tests/functional/lang/eval-fail-deepseq.err.exp      |  6 ------
 .../lang/eval-fail-dup-dynamic-attrs.err.exp         |  4 ----
 ...al-fail-foldlStrict-strict-op-application.err.exp |  8 --------
 .../lang/eval-fail-fromTOML-timestamps.err.exp       |  2 --
 .../lang/eval-fail-hashfile-missing.err.exp          |  2 --
 tests/functional/lang/eval-fail-list.err.exp         |  2 --
 tests/functional/lang/eval-fail-missing-arg.err.exp  |  4 ----
 tests/functional/lang/eval-fail-not-throws.err.exp   |  4 ----
 tests/functional/lang/eval-fail-path-slash.err.exp   |  2 --
 tests/functional/lang/eval-fail-recursion.err.exp    |  4 ----
 tests/functional/lang/eval-fail-remove.err.exp       |  4 ----
 tests/functional/lang/eval-fail-scope-5.err.exp      |  8 --------
 tests/functional/lang/eval-fail-seq.err.exp          |  4 ----
 tests/functional/lang/eval-fail-set.err.exp          |  2 --
 tests/functional/lang/eval-fail-substring.err.exp    |  2 --
 tests/functional/lang/eval-fail-to-path.err.exp      |  2 --
 tests/functional/lang/eval-fail-toJSON.err.exp       | 12 ------------
 .../functional/lang/eval-fail-undeclared-arg.err.exp |  4 ----
 .../lang/eval-fail-using-set-as-attr-name.err.exp    |  2 --
 tests/functional/lang/parse-fail-dup-attrs-1.err.exp |  2 --
 tests/functional/lang/parse-fail-dup-attrs-2.err.exp |  2 --
 tests/functional/lang/parse-fail-dup-attrs-3.err.exp |  2 --
 tests/functional/lang/parse-fail-dup-attrs-4.err.exp |  2 --
 tests/functional/lang/parse-fail-dup-attrs-7.err.exp |  2 --
 tests/functional/lang/parse-fail-dup-formals.err.exp |  2 --
 .../functional/lang/parse-fail-eof-in-string.err.exp |  2 --
 .../lang/parse-fail-mixed-nested-attrs1.err.exp      |  2 --
 .../lang/parse-fail-mixed-nested-attrs2.err.exp      |  2 --
 tests/functional/lang/parse-fail-patterns-1.err.exp  |  2 --
 .../lang/parse-fail-regression-20060610.err.exp      |  2 --
 tests/functional/lang/parse-fail-undef-var-2.err.exp |  2 --
 tests/functional/lang/parse-fail-undef-var.err.exp   |  2 --
 tests/functional/lang/parse-fail-utf8.err.exp        |  2 --
 45 files changed, 1 insertion(+), 140 deletions(-)

diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 72c346cb5..bc0194d59 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -173,10 +173,9 @@ static bool printUnknownLocations = getEnv("_NIX_EVAL_SHOW_UNKNOWN_LOCATIONS").h
 static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std::shared_ptr<AbstractPos> & pos) {
     bool hasPos = pos && *pos;
     if (hasPos) {
-        oss << "\n" << indent << ANSI_BLUE << "at " ANSI_WARNING << *pos << ANSI_NORMAL << ":";
+        oss << indent << ANSI_BLUE << "at " ANSI_WARNING << *pos << ANSI_NORMAL << ":";
 
         if (auto loc = pos->getCodeLines()) {
-            oss << "\n";
             printCodeLines(oss, "", *pos, *loc);
             oss << "\n";
         }
diff --git a/tests/functional/lang/eval-fail-abort.err.exp b/tests/functional/lang/eval-fail-abort.err.exp
index 345232d3f..20e7b9e18 100644
--- a/tests/functional/lang/eval-fail-abort.err.exp
+++ b/tests/functional/lang/eval-fail-abort.err.exp
@@ -1,8 +1,6 @@
 error:
        … while calling the 'abort' builtin
-
          at /pwd/lang/eval-fail-abort.nix:1:14:
-
             1| if true then abort "this should fail" else 1
              |              ^
             2|
diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp
index ad91a22aa..37e0bd9ee 100644
--- a/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp
+++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp
@@ -1,8 +1,6 @@
 error:
        … while calling the 'addDrvOutputDependencies' builtin
-
          at /pwd/lang/eval-fail-addDrvOutputDependencies-empty-context.nix:1:1:
-
             1| builtins.addDrvOutputDependencies ""
              | ^
             2|
diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp
index bb389db4e..6828e03c8 100644
--- a/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp
+++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp
@@ -1,8 +1,6 @@
 error:
        … while calling the 'addDrvOutputDependencies' builtin
-
          at /pwd/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix:18:4:
-
            17|
            18| in builtins.addDrvOutputDependencies combo-path
              |    ^
diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp
index 070381118..72b5e6368 100644
--- a/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp
+++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp
@@ -1,8 +1,6 @@
 error:
        … while calling the 'addDrvOutputDependencies' builtin
-
          at /pwd/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix:9:4:
-
             8|
             9| in builtins.addDrvOutputDependencies drv.outPath
              |    ^
diff --git a/tests/functional/lang/eval-fail-assert.err.exp b/tests/functional/lang/eval-fail-assert.err.exp
index aeecd8167..0656ec81c 100644
--- a/tests/functional/lang/eval-fail-assert.err.exp
+++ b/tests/functional/lang/eval-fail-assert.err.exp
@@ -1,35 +1,27 @@
 error:
        … while evaluating the attribute 'body'
-
          at /pwd/lang/eval-fail-assert.nix:4:3:
-
             3|
             4|   body = x "x";
              |   ^
             5| }
 
        … from call site
-
          at /pwd/lang/eval-fail-assert.nix:4:10:
-
             3|
             4|   body = x "x";
              |          ^
             5| }
 
        … while calling 'x'
-
          at /pwd/lang/eval-fail-assert.nix:2:7:
-
             1| let {
             2|   x = arg: assert arg == "y"; 123;
              |       ^
             3|
 
        error: assertion '(arg == "y")' failed
-
        at /pwd/lang/eval-fail-assert.nix:2:12:
-
             1| let {
             2|   x = arg: assert arg == "y"; 123;
              |            ^
diff --git a/tests/functional/lang/eval-fail-attr-name-type.err.exp b/tests/functional/lang/eval-fail-attr-name-type.err.exp
index 5f9a073dd..23cceb58a 100644
--- a/tests/functional/lang/eval-fail-attr-name-type.err.exp
+++ b/tests/functional/lang/eval-fail-attr-name-type.err.exp
@@ -1,17 +1,13 @@
 error:
        … while evaluating the attribute 'puppy."${key}"'
-
          at /pwd/lang/eval-fail-attr-name-type.nix:3:5:
-
             2|   attrs = {
             3|     puppy.doggy = {};
              |     ^
             4|   };
 
        … while evaluating an attribute name
-
          at /pwd/lang/eval-fail-attr-name-type.nix:7:17:
-
             6| in
             7|   attrs.puppy.${key}
              |                 ^
diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp
index eb73e9a52..b461b2e02 100644
--- a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp
+++ b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp
@@ -1,8 +1,6 @@
 error:
        … while evaluating a path segment
-
          at /pwd/lang/eval-fail-bad-string-interpolation-1.nix:1:2:
-
             1| "${x: x}"
              |  ^
             2|
diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp
index ac14f329b..95f4c2460 100644
--- a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp
+++ b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp
@@ -1,8 +1,6 @@
 error:
        … while evaluating a path segment
-
          at /pwd/lang/eval-fail-bad-string-interpolation-3.nix:1:3:
-
             1| ''${x: x}''
              |   ^
             2|
diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp
index 07843a480..4950f8ddb 100644
--- a/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp
+++ b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp
@@ -1,8 +1,6 @@
 error:
        … while evaluating a path segment
-
          at /pwd/lang/eval-fail-bad-string-interpolation-4.nix:9:3:
-
             8| # The error message should not be too long.
             9| ''${pkgs}''
              |   ^
diff --git a/tests/functional/lang/eval-fail-blackhole.err.exp b/tests/functional/lang/eval-fail-blackhole.err.exp
index f0618d8ac..95e33a5fe 100644
--- a/tests/functional/lang/eval-fail-blackhole.err.exp
+++ b/tests/functional/lang/eval-fail-blackhole.err.exp
@@ -1,17 +1,13 @@
 error:
        … while evaluating the attribute 'body'
-
          at /pwd/lang/eval-fail-blackhole.nix:2:3:
-
             1| let {
             2|   body = x;
              |   ^
             3|   x = y;
 
        error: infinite recursion encountered
-
        at /pwd/lang/eval-fail-blackhole.nix:3:7:
-
             2|   body = x;
             3|   x = y;
              |       ^
diff --git a/tests/functional/lang/eval-fail-call-primop.err.exp b/tests/functional/lang/eval-fail-call-primop.err.exp
index 19b407c47..ae5b55ed4 100644
--- a/tests/functional/lang/eval-fail-call-primop.err.exp
+++ b/tests/functional/lang/eval-fail-call-primop.err.exp
@@ -1,8 +1,6 @@
 error:
        … while calling the 'length' builtin
-
          at /pwd/lang/eval-fail-call-primop.nix:1:1:
-
             1| builtins.length 1
              | ^
             2|
diff --git a/tests/functional/lang/eval-fail-deepseq.err.exp b/tests/functional/lang/eval-fail-deepseq.err.exp
index 5e204ba73..11b62340d 100644
--- a/tests/functional/lang/eval-fail-deepseq.err.exp
+++ b/tests/functional/lang/eval-fail-deepseq.err.exp
@@ -1,24 +1,18 @@
 error:
        … while calling the 'deepSeq' builtin
-
          at /pwd/lang/eval-fail-deepseq.nix:1:1:
-
             1| builtins.deepSeq { x = abort "foo"; } 456
              | ^
             2|
 
        … while evaluating the attribute 'x'
-
          at /pwd/lang/eval-fail-deepseq.nix:1:20:
-
             1| builtins.deepSeq { x = abort "foo"; } 456
              |                    ^
             2|
 
        … while calling the 'abort' builtin
-
          at /pwd/lang/eval-fail-deepseq.nix:1:24:
-
             1| builtins.deepSeq { x = abort "foo"; } 456
              |                        ^
             2|
diff --git a/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp b/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp
index c5fa67523..834f9c67b 100644
--- a/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp
+++ b/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp
@@ -1,17 +1,13 @@
 error:
        … while evaluating the attribute 'set'
-
          at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:3:
-
             1| {
             2|   set = { "${"" + "b"}" = 1; };
              |   ^
             3|   set = { "${"b" + ""}" = 2; };
 
        error: dynamic attribute 'b' already defined at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:11
-
        at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:3:11:
-
             2|   set = { "${"" + "b"}" = 1; };
             3|   set = { "${"b" + ""}" = 2; };
              |           ^
diff --git a/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp b/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp
index 0069285fb..7cb08af8a 100644
--- a/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp
+++ b/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp
@@ -1,35 +1,27 @@
 error:
        … while calling the 'foldl'' builtin
-
          at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:2:1:
-
             1| # Tests that the result of applying op is forced even if the value is never used
             2| builtins.foldl'
              | ^
             3|   (_: f: f null)
 
        … while calling anonymous lambda
-
          at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:3:7:
-
             2| builtins.foldl'
             3|   (_: f: f null)
              |       ^
             4|   null
 
        … from call site
-
          at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:3:10:
-
             2| builtins.foldl'
             3|   (_: f: f null)
              |          ^
             4|   null
 
        … while calling anonymous lambda
-
          at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:5:6:
-
             4|   null
             5|   [ (_: throw "Not the final value, but is still forced!") (_: 23) ]
              |      ^
diff --git a/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp b/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp
index 5b60d253d..73f9df8cc 100644
--- a/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp
+++ b/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp
@@ -1,8 +1,6 @@
 error:
        … while calling the 'fromTOML' builtin
-
          at /pwd/lang/eval-fail-fromTOML-timestamps.nix:1:1:
-
             1| builtins.fromTOML ''
              | ^
             2|   key = "value"
diff --git a/tests/functional/lang/eval-fail-hashfile-missing.err.exp b/tests/functional/lang/eval-fail-hashfile-missing.err.exp
index 6d38608c0..1e4653927 100644
--- a/tests/functional/lang/eval-fail-hashfile-missing.err.exp
+++ b/tests/functional/lang/eval-fail-hashfile-missing.err.exp
@@ -1,8 +1,6 @@
 error:
        … while calling the 'toString' builtin
-
          at /pwd/lang/eval-fail-hashfile-missing.nix:4:3:
-
             3| in
             4|   toString (builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"]))
              |   ^
diff --git a/tests/functional/lang/eval-fail-list.err.exp b/tests/functional/lang/eval-fail-list.err.exp
index 24d682118..4320fc022 100644
--- a/tests/functional/lang/eval-fail-list.err.exp
+++ b/tests/functional/lang/eval-fail-list.err.exp
@@ -1,8 +1,6 @@
 error:
        … while evaluating one of the elements to concatenate
-
          at /pwd/lang/eval-fail-list.nix:1:2:
-
             1| 8++1
              |  ^
             2|
diff --git a/tests/functional/lang/eval-fail-missing-arg.err.exp b/tests/functional/lang/eval-fail-missing-arg.err.exp
index 61fabf0d5..3b162fe1b 100644
--- a/tests/functional/lang/eval-fail-missing-arg.err.exp
+++ b/tests/functional/lang/eval-fail-missing-arg.err.exp
@@ -1,16 +1,12 @@
 error:
        … from call site
-
          at /pwd/lang/eval-fail-missing-arg.nix:1:1:
-
             1| ({x, y, z}: x + y + z) {x = "foo"; z = "bar";}
              | ^
             2|
 
        error: function 'anonymous lambda' called without required argument 'y'
-
        at /pwd/lang/eval-fail-missing-arg.nix:1:2:
-
             1| ({x, y, z}: x + y + z) {x = "foo"; z = "bar";}
              |  ^
             2|
diff --git a/tests/functional/lang/eval-fail-not-throws.err.exp b/tests/functional/lang/eval-fail-not-throws.err.exp
index b290afb0a..fc81f7277 100644
--- a/tests/functional/lang/eval-fail-not-throws.err.exp
+++ b/tests/functional/lang/eval-fail-not-throws.err.exp
@@ -1,16 +1,12 @@
 error:
        … in the argument of the not operator
-
          at /pwd/lang/eval-fail-not-throws.nix:1:4:
-
             1| ! (throw "uh oh!")
              |    ^
             2|
 
        … while calling the 'throw' builtin
-
          at /pwd/lang/eval-fail-not-throws.nix:1:4:
-
             1| ! (throw "uh oh!")
              |    ^
             2|
diff --git a/tests/functional/lang/eval-fail-path-slash.err.exp b/tests/functional/lang/eval-fail-path-slash.err.exp
index f0011c97f..e3531d352 100644
--- a/tests/functional/lang/eval-fail-path-slash.err.exp
+++ b/tests/functional/lang/eval-fail-path-slash.err.exp
@@ -1,7 +1,5 @@
 error: path has a trailing slash
-
        at /pwd/lang/eval-fail-path-slash.nix:6:12:
-
             5| # and https://nixos.org/nix-dev/2016-June/020829.html
             6| /nix/store/
              |            ^
diff --git a/tests/functional/lang/eval-fail-recursion.err.exp b/tests/functional/lang/eval-fail-recursion.err.exp
index af64133cb..19380dc65 100644
--- a/tests/functional/lang/eval-fail-recursion.err.exp
+++ b/tests/functional/lang/eval-fail-recursion.err.exp
@@ -1,16 +1,12 @@
 error:
        … in the right operand of the update (//) operator
-
          at /pwd/lang/eval-fail-recursion.nix:1:12:
-
             1| let a = {} // a; in a.foo
              |            ^
             2|
 
        error: infinite recursion encountered
-
        at /pwd/lang/eval-fail-recursion.nix:1:15:
-
             1| let a = {} // a; in a.foo
              |               ^
             2|
diff --git a/tests/functional/lang/eval-fail-remove.err.exp b/tests/functional/lang/eval-fail-remove.err.exp
index e82cdac98..292b3c3f3 100644
--- a/tests/functional/lang/eval-fail-remove.err.exp
+++ b/tests/functional/lang/eval-fail-remove.err.exp
@@ -1,17 +1,13 @@
 error:
        … while evaluating the attribute 'body'
-
          at /pwd/lang/eval-fail-remove.nix:4:3:
-
             3|
             4|   body = (removeAttrs attrs ["x"]).x;
              |   ^
             5| }
 
        error: attribute 'x' missing
-
        at /pwd/lang/eval-fail-remove.nix:4:10:
-
             3|
             4|   body = (removeAttrs attrs ["x"]).x;
              |          ^
diff --git a/tests/functional/lang/eval-fail-scope-5.err.exp b/tests/functional/lang/eval-fail-scope-5.err.exp
index 22b6166f8..b0b05cad7 100644
--- a/tests/functional/lang/eval-fail-scope-5.err.exp
+++ b/tests/functional/lang/eval-fail-scope-5.err.exp
@@ -1,35 +1,27 @@
 error:
        … while evaluating the attribute 'body'
-
          at /pwd/lang/eval-fail-scope-5.nix:8:3:
-
             7|
             8|   body = f {};
              |   ^
             9|
 
        … from call site
-
          at /pwd/lang/eval-fail-scope-5.nix:8:10:
-
             7|
             8|   body = f {};
              |          ^
             9|
 
        … while calling 'f'
-
          at /pwd/lang/eval-fail-scope-5.nix:6:7:
-
             5|
             6|   f = {x ? y, y ? x}: x + y;
              |       ^
             7|
 
        error: infinite recursion encountered
-
        at /pwd/lang/eval-fail-scope-5.nix:6:12:
-
             5|
             6|   f = {x ? y, y ? x}: x + y;
              |            ^
diff --git a/tests/functional/lang/eval-fail-seq.err.exp b/tests/functional/lang/eval-fail-seq.err.exp
index 33a7e9491..3e3d71b15 100644
--- a/tests/functional/lang/eval-fail-seq.err.exp
+++ b/tests/functional/lang/eval-fail-seq.err.exp
@@ -1,16 +1,12 @@
 error:
        … while calling the 'seq' builtin
-
          at /pwd/lang/eval-fail-seq.nix:1:1:
-
             1| builtins.seq (abort "foo") 2
              | ^
             2|
 
        … while calling the 'abort' builtin
-
          at /pwd/lang/eval-fail-seq.nix:1:15:
-
             1| builtins.seq (abort "foo") 2
              |               ^
             2|
diff --git a/tests/functional/lang/eval-fail-set.err.exp b/tests/functional/lang/eval-fail-set.err.exp
index 0d0140508..6dd646e11 100644
--- a/tests/functional/lang/eval-fail-set.err.exp
+++ b/tests/functional/lang/eval-fail-set.err.exp
@@ -1,7 +1,5 @@
 error: undefined variable 'x'
-
        at /pwd/lang/eval-fail-set.nix:1:3:
-
             1| 8.x
              |   ^
             2|
diff --git a/tests/functional/lang/eval-fail-substring.err.exp b/tests/functional/lang/eval-fail-substring.err.exp
index 5c58be29a..0457a826e 100644
--- a/tests/functional/lang/eval-fail-substring.err.exp
+++ b/tests/functional/lang/eval-fail-substring.err.exp
@@ -1,8 +1,6 @@
 error:
        … while calling the 'substring' builtin
-
          at /pwd/lang/eval-fail-substring.nix:1:1:
-
             1| builtins.substring (builtins.sub 0 1) 1 "x"
              | ^
             2|
diff --git a/tests/functional/lang/eval-fail-to-path.err.exp b/tests/functional/lang/eval-fail-to-path.err.exp
index 4ffa2cf6d..d6b17be99 100644
--- a/tests/functional/lang/eval-fail-to-path.err.exp
+++ b/tests/functional/lang/eval-fail-to-path.err.exp
@@ -1,8 +1,6 @@
 error:
        … while calling the 'toPath' builtin
-
          at /pwd/lang/eval-fail-to-path.nix:1:1:
-
             1| builtins.toPath "foo/bar"
              | ^
             2|
diff --git a/tests/functional/lang/eval-fail-toJSON.err.exp b/tests/functional/lang/eval-fail-toJSON.err.exp
index 4e618c203..4f6003437 100644
--- a/tests/functional/lang/eval-fail-toJSON.err.exp
+++ b/tests/functional/lang/eval-fail-toJSON.err.exp
@@ -1,25 +1,19 @@
 error:
        … while calling the 'toJSON' builtin
-
          at /pwd/lang/eval-fail-toJSON.nix:1:1:
-
             1| builtins.toJSON {
              | ^
             2|   a.b = [
 
        … while evaluating attribute 'a'
-
          at /pwd/lang/eval-fail-toJSON.nix:2:3:
-
             1| builtins.toJSON {
             2|   a.b = [
              |   ^
             3|     true
 
        … while evaluating attribute 'b'
-
          at /pwd/lang/eval-fail-toJSON.nix:2:3:
-
             1| builtins.toJSON {
             2|   a.b = [
              |   ^
@@ -28,27 +22,21 @@ error:
        … while evaluating list element at index 3
 
        … while evaluating attribute 'c'
-
          at /pwd/lang/eval-fail-toJSON.nix:7:7:
-
             6|     {
             7|       c.d = throw "hah no";
              |       ^
             8|     }
 
        … while evaluating attribute 'd'
-
          at /pwd/lang/eval-fail-toJSON.nix:7:7:
-
             6|     {
             7|       c.d = throw "hah no";
              |       ^
             8|     }
 
        … while calling the 'throw' builtin
-
          at /pwd/lang/eval-fail-toJSON.nix:7:13:
-
             6|     {
             7|       c.d = throw "hah no";
              |             ^
diff --git a/tests/functional/lang/eval-fail-undeclared-arg.err.exp b/tests/functional/lang/eval-fail-undeclared-arg.err.exp
index 30db743c7..6e13a138e 100644
--- a/tests/functional/lang/eval-fail-undeclared-arg.err.exp
+++ b/tests/functional/lang/eval-fail-undeclared-arg.err.exp
@@ -1,16 +1,12 @@
 error:
        … from call site
-
          at /pwd/lang/eval-fail-undeclared-arg.nix:1:1:
-
             1| ({x, z}: x + z) {x = "foo"; y = "bla"; z = "bar";}
              | ^
             2|
 
        error: function 'anonymous lambda' called with unexpected argument 'y'
-
        at /pwd/lang/eval-fail-undeclared-arg.nix:1:2:
-
             1| ({x, z}: x + z) {x = "foo"; y = "bla"; z = "bar";}
              |  ^
             2|
diff --git a/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp b/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp
index 811d01b03..0a4f56ac5 100644
--- a/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp
+++ b/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp
@@ -1,8 +1,6 @@
 error:
        … while evaluating an attribute name
-
          at /pwd/lang/eval-fail-using-set-as-attr-name.nix:5:10:
-
             4| in
             5|   attr.${key}
              |          ^
diff --git a/tests/functional/lang/parse-fail-dup-attrs-1.err.exp b/tests/functional/lang/parse-fail-dup-attrs-1.err.exp
index 4fe6b7a1f..6c3a3510c 100644
--- a/tests/functional/lang/parse-fail-dup-attrs-1.err.exp
+++ b/tests/functional/lang/parse-fail-dup-attrs-1.err.exp
@@ -1,7 +1,5 @@
 error: attribute 'x' already defined at «stdin»:1:3
-
        at «stdin»:3:3:
-
             2|   y = 456;
             3|   x = 789;
              |   ^
diff --git a/tests/functional/lang/parse-fail-dup-attrs-2.err.exp b/tests/functional/lang/parse-fail-dup-attrs-2.err.exp
index 3aba2891f..fecdece20 100644
--- a/tests/functional/lang/parse-fail-dup-attrs-2.err.exp
+++ b/tests/functional/lang/parse-fail-dup-attrs-2.err.exp
@@ -1,7 +1,5 @@
 error: attribute 'x' already defined at «stdin»:9:5
-
        at «stdin»:10:17:
-
             9|     x = 789;
            10|     inherit (as) x;
              |                 ^
diff --git a/tests/functional/lang/parse-fail-dup-attrs-3.err.exp b/tests/functional/lang/parse-fail-dup-attrs-3.err.exp
index 3aba2891f..fecdece20 100644
--- a/tests/functional/lang/parse-fail-dup-attrs-3.err.exp
+++ b/tests/functional/lang/parse-fail-dup-attrs-3.err.exp
@@ -1,7 +1,5 @@
 error: attribute 'x' already defined at «stdin»:9:5
-
        at «stdin»:10:17:
-
             9|     x = 789;
            10|     inherit (as) x;
              |                 ^
diff --git a/tests/functional/lang/parse-fail-dup-attrs-4.err.exp b/tests/functional/lang/parse-fail-dup-attrs-4.err.exp
index ff68446a1..f85ffea51 100644
--- a/tests/functional/lang/parse-fail-dup-attrs-4.err.exp
+++ b/tests/functional/lang/parse-fail-dup-attrs-4.err.exp
@@ -1,7 +1,5 @@
 error: attribute 'services.ssh.port' already defined at «stdin»:2:3
-
        at «stdin»:3:3:
-
             2|   services.ssh.port = 22;
             3|   services.ssh.port = 23;
              |   ^
diff --git a/tests/functional/lang/parse-fail-dup-attrs-7.err.exp b/tests/functional/lang/parse-fail-dup-attrs-7.err.exp
index 512a499ca..98cea9dae 100644
--- a/tests/functional/lang/parse-fail-dup-attrs-7.err.exp
+++ b/tests/functional/lang/parse-fail-dup-attrs-7.err.exp
@@ -1,7 +1,5 @@
 error: attribute 'x' already defined at «stdin»:6:12
-
        at «stdin»:7:12:
-
             6|     inherit x;
             7|     inherit x;
              |            ^
diff --git a/tests/functional/lang/parse-fail-dup-formals.err.exp b/tests/functional/lang/parse-fail-dup-formals.err.exp
index 1d566fb33..d7c7e0237 100644
--- a/tests/functional/lang/parse-fail-dup-formals.err.exp
+++ b/tests/functional/lang/parse-fail-dup-formals.err.exp
@@ -1,6 +1,4 @@
 error: duplicate formal function argument 'x'
-
        at «stdin»:1:8:
-
             1| {x, y, x}: x
              |        ^
diff --git a/tests/functional/lang/parse-fail-eof-in-string.err.exp b/tests/functional/lang/parse-fail-eof-in-string.err.exp
index f9fa72312..b28d35950 100644
--- a/tests/functional/lang/parse-fail-eof-in-string.err.exp
+++ b/tests/functional/lang/parse-fail-eof-in-string.err.exp
@@ -1,7 +1,5 @@
 error: syntax error, unexpected end of file, expecting '"'
-
        at «stdin»:3:5:
-
             2| # Note that this file must not end with a newline.
             3| a 1"$
              |     ^
diff --git a/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp
index 32f776795..a4472156b 100644
--- a/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp
+++ b/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp
@@ -1,7 +1,5 @@
 error: attribute 'z' already defined at «stdin»:3:16
-
        at «stdin»:2:3:
-
             1| {
             2|   x.z = 3;
              |   ^
diff --git a/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp
index 0437cd50c..ead1f0dbd 100644
--- a/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp
+++ b/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp
@@ -1,7 +1,5 @@
 error: attribute 'y' already defined at «stdin»:3:9
-
        at «stdin»:2:3:
-
             1| {
             2|   x.y.y = 3;
              |   ^
diff --git a/tests/functional/lang/parse-fail-patterns-1.err.exp b/tests/functional/lang/parse-fail-patterns-1.err.exp
index 634a04aaa..6ba39d884 100644
--- a/tests/functional/lang/parse-fail-patterns-1.err.exp
+++ b/tests/functional/lang/parse-fail-patterns-1.err.exp
@@ -1,7 +1,5 @@
 error: duplicate formal function argument 'args'
-
        at «stdin»:1:1:
-
             1| args@{args, x, y, z}: x
              | ^
             2|
diff --git a/tests/functional/lang/parse-fail-regression-20060610.err.exp b/tests/functional/lang/parse-fail-regression-20060610.err.exp
index 167d01e85..d8875a6a5 100644
--- a/tests/functional/lang/parse-fail-regression-20060610.err.exp
+++ b/tests/functional/lang/parse-fail-regression-20060610.err.exp
@@ -1,7 +1,5 @@
 error: undefined variable 'gcc'
-
        at «stdin»:8:12:
-
             7|
             8|   body = ({
              |            ^
diff --git a/tests/functional/lang/parse-fail-undef-var-2.err.exp b/tests/functional/lang/parse-fail-undef-var-2.err.exp
index 77c96bbd2..a58d8dca4 100644
--- a/tests/functional/lang/parse-fail-undef-var-2.err.exp
+++ b/tests/functional/lang/parse-fail-undef-var-2.err.exp
@@ -1,7 +1,5 @@
 error: syntax error, unexpected ':', expecting '}'
-
        at «stdin»:3:13:
-
             2|
             3|   f = {x, y :
              |             ^
diff --git a/tests/functional/lang/parse-fail-undef-var.err.exp b/tests/functional/lang/parse-fail-undef-var.err.exp
index 48e88747f..3d143d9af 100644
--- a/tests/functional/lang/parse-fail-undef-var.err.exp
+++ b/tests/functional/lang/parse-fail-undef-var.err.exp
@@ -1,7 +1,5 @@
 error: undefined variable 'y'
-
        at «stdin»:1:4:
-
             1| x: y
              |    ^
             2|
diff --git a/tests/functional/lang/parse-fail-utf8.err.exp b/tests/functional/lang/parse-fail-utf8.err.exp
index 6087479a3..e83abdb9e 100644
--- a/tests/functional/lang/parse-fail-utf8.err.exp
+++ b/tests/functional/lang/parse-fail-utf8.err.exp
@@ -1,6 +1,4 @@
 error: syntax error, unexpected invalid token, expecting end of file
-
        at «stdin»:1:5:
-
             1| 123 �
              |     ^

From 7f5ed330e40d0aa2a2f907b2d4157329ff953cd2 Mon Sep 17 00:00:00 2001
From: Rebecca Turner <rbt@sent.as>
Date: Sat, 16 Dec 2023 07:05:31 -0500
Subject: [PATCH 244/421] Document `Makefile` variables in `hacking.md` (#9620)

---
 doc/manual/src/contributing/hacking.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index 4d3d66397..421ac981c 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -64,6 +64,15 @@ $ nix build
 
 You can also build Nix for one of the [supported platforms](#platforms).
 
+## Makefile variables
+
+- `ENABLE_BUILD=yes` to enable building the C++ code.
+- `ENABLE_TESTS=yes` to enable building the tests.
+- `OPTIMIZE=1` to enable optimizations.
+- `doc_generate=yes` to enable building the documentation (manual, man pages, etc.).
+
+  The docs can take a while to build, so you may want to disable this for local development.
+
 ## Building Nix
 
 To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found:

From c05d4fadd5f0943de0a00b17c85626d73152da66 Mon Sep 17 00:00:00 2001
From: David Arnold <dgx.arnold@gmail.com>
Date: Sat, 16 Dec 2023 23:07:17 +0100
Subject: [PATCH 245/421] fix: valid branch name

---
 src/libutil/url-parts.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh
index 07bc8d0cd..e968eea4b 100644
--- a/src/libutil/url-parts.hh
+++ b/src/libutil/url-parts.hh
@@ -25,7 +25,7 @@ const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRege
 
 /// A Git ref (i.e. branch or tag name).
 /// \todo check that this is correct.
-const static std::string refRegexS = "[a-zA-Z0-9@][a-zA-Z0-9_.\\/@-]*";
+const static std::string refRegexS = "[a-zA-Z0-9@][a-zA-Z0-9_.\\/@-+]*";
 extern std::regex refRegex;
 
 /// Instead of defining what a good Git Ref is, we define what a bad Git Ref is

From 06bed2eacdeaa3b92d6e35c5d2133c31baa9e56f Mon Sep 17 00:00:00 2001
From: Julia Evans <julia@jvns.ca>
Date: Sun, 17 Dec 2023 12:00:50 -0500
Subject: [PATCH 246/421] Make fetchTree locked input error message clearer

---
 src/libexpr/primops/fetchTree.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index eb2df8626..fa503665e 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -167,7 +167,10 @@ static void fetchTree(
         input = lookupInRegistries(state.store, input).first;
 
     if (evalSettings.pureEval && !input.isLocked())
-        state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", state.positions[pos]));
+        if (type == "git")
+            state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchGit' requires a locked input, at %s", state.positions[pos]));
+        else
+            state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", state.positions[pos]));
 
     state.checkURI(input.toURLString());
 

From a47fabff0dbcd63e2645db7336dde5865a1995c4 Mon Sep 17 00:00:00 2001
From: Julia Evans <julia@jvns.ca>
Date: Sun, 17 Dec 2023 12:14:55 -0500
Subject: [PATCH 247/421] use params.isFetchGit instead to check if it came
 from fetchGit

---
 src/libexpr/primops/fetchTree.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index fa503665e..505900b30 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -167,7 +167,7 @@ static void fetchTree(
         input = lookupInRegistries(state.store, input).first;
 
     if (evalSettings.pureEval && !input.isLocked())
-        if (type == "git")
+        if (params.isFetchGit)
             state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchGit' requires a locked input, at %s", state.positions[pos]));
         else
             state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", state.positions[pos]));

From 4f9580085441a4255ce746a4cc498b45cc25a899 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Mon, 18 Dec 2023 11:41:52 +0100
Subject: [PATCH 248/421] add cross-reference

---
 src/libstore/globals.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index e28615cdc..b35dc37a1 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -631,7 +631,7 @@ public:
 
           At least one of the following condition must be met
           for Nix to accept copying a store object from another
-          Nix store (such as a substituter):
+          Nix store (such as a [substituter](#conf-substituters)):
 
           - the store object has been signed using a key in the trusted keys list
           - the [`require-sigs`](#conf-require-sigs) option has been set to `false`

From d19a6675286a38edf8970459cbf454322f8151cb Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Mon, 18 Dec 2023 13:54:40 +0100
Subject: [PATCH 249/421] CODEOWNERS: unsubscribe fricklerhandwerk (#9614)

---
 .github/CODEOWNERS | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 39d595199..526fecabf 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -10,16 +10,8 @@
 # This file
 .github/CODEOWNERS @edolstra
 
-# Public documentation
-/doc @fricklerhandwerk
-*.md @fricklerhandwerk
-
 # Documentation of built-in functions
-src/libexpr/primops.cc @fricklerhandwerk @roberth
-# Documentation on experimental features
-src/libutil/experimental-features.cc @fricklerhandwerk
-# Documentation on configuration settings
-src/libstore/globals.hh @fricklerhandwerk
+src/libexpr/primops.cc @roberth
 
 # Libstore layer
 /src/libstore @thufschmitt

From dfc876531f269950a4e183a4f77a813c421d7d64 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 4 Nov 2023 16:25:41 -0400
Subject: [PATCH 250/421] Organize content addressing, use `SourceAccessor`
 with `Store::addToStore`

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 perl/lib/Nix/Store.xs                       | 12 ++-
 src/libcmd/installable-value.cc             |  2 +-
 src/libexpr/eval.cc                         |  2 +-
 src/libexpr/primops.cc                      |  2 +-
 src/libfetchers/cache.cc                    | 16 ++--
 src/libfetchers/cache.hh                    |  6 +-
 src/libfetchers/fetchers.cc                 |  2 +-
 src/libfetchers/git.cc                      |  4 +-
 src/libfetchers/github.cc                   |  4 +-
 src/libfetchers/input-accessor.cc           | 39 ++++++---
 src/libfetchers/input-accessor.hh           |  8 +-
 src/libfetchers/mercurial.cc                | 25 ++++--
 src/libfetchers/tarball.cc                  | 14 +--
 src/libstore/binary-cache-store.cc          | 56 ++++++------
 src/libstore/binary-cache-store.hh          | 24 ++++--
 src/libstore/build/local-derivation-goal.cc | 90 +++++++++----------
 src/libstore/build/worker.cc                |  4 +-
 src/libstore/content-address.cc             | 28 ++++--
 src/libstore/content-address.hh             | 38 ++++-----
 src/libstore/daemon.cc                      | 19 +----
 src/libstore/legacy-ssh-store.hh            | 15 ++--
 src/libstore/local-store.cc                 | 95 ++++++++-------------
 src/libstore/local-store.hh                 | 22 ++---
 src/libstore/optimise-store.cc              | 15 +++-
 src/libstore/remote-store.cc                |  9 +-
 src/libstore/remote-store.hh                | 11 ++-
 src/libstore/store-api.cc                   | 90 ++++++++++---------
 src/libstore/store-api.hh                   | 36 +++++---
 src/libstore/store-dir-config.hh            | 14 +--
 src/libutil/file-content-address.cc         | 49 +++++++++++
 src/libutil/file-content-address.hh         | 56 ++++++++++++
 src/libutil/hash.cc                         |  9 --
 src/libutil/hash.hh                         |  7 +-
 src/nix-store/nix-store.cc                  | 20 ++++-
 src/nix/add-to-store.cc                     | 42 +++------
 src/nix/hash.cc                             | 11 +--
 src/nix/prefetch.cc                         |  7 +-
 tests/unit/libexpr/primops.cc               |  2 +-
 38 files changed, 515 insertions(+), 390 deletions(-)
 create mode 100644 src/libutil/file-content-address.cc
 create mode 100644 src/libutil/file-content-address.hh

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index 82c7db608..4964b8a34 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -13,6 +13,7 @@
 #include "globals.hh"
 #include "store-api.hh"
 #include "crypto.hh"
+#include "posix-source-accessor.hh"
 
 #include <sodium.h>
 #include <nlohmann/json.hpp>
@@ -205,7 +206,10 @@ void importPaths(int fd, int dontCheckSigs)
 SV * hashPath(char * algo, int base32, char * path)
     PPCODE:
         try {
-            Hash h = hashPath(parseHashAlgo(algo), path).first;
+            PosixSourceAccessor accessor;
+            Hash h = hashPath(
+                accessor, CanonPath::fromCwd(path),
+                FileIngestionMethod::Recursive, parseHashAlgo(algo)).first;
             auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
@@ -281,7 +285,11 @@ SV * addToStore(char * srcPath, int recursive, char * algo)
     PPCODE:
         try {
             auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-            auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashAlgo(algo));
+            PosixSourceAccessor accessor;
+            auto path = store()->addToStore(
+                std::string(baseNameOf(srcPath)),
+                accessor, CanonPath::fromCwd(srcPath),
+                method, parseHashAlgo(algo));
             XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc
index 08ad35105..bdc34bbe3 100644
--- a/src/libcmd/installable-value.cc
+++ b/src/libcmd/installable-value.cc
@@ -44,7 +44,7 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
 std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
 {
     if (v.type() == nPath) {
-        auto storePath = v.path().fetchToStore(state->store);
+        auto storePath = v.path().fetchToStore(*state->store);
         return {{
             .path = DerivedPath::Opaque {
                 .path = std::move(storePath),
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 1552e3e92..c9c25c898 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -2317,7 +2317,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
     auto dstPath = i != srcToStore.end()
         ? i->second
         : [&]() {
-            auto dstPath = path.fetchToStore(store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair);
+            auto dstPath = path.fetchToStore(*store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair);
             allowPath(dstPath);
             srcToStore.insert_or_assign(path, dstPath);
             printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index d78a28c73..75ee1e38d 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -2229,7 +2229,7 @@ static void addPath(
             });
 
         if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
-            auto dstPath = path.fetchToStore(state.store, name, method, filter.get(), state.repair);
+            auto dstPath = path.fetchToStore(*state.store, name, method, filter.get(), state.repair);
             if (expectedHash && expectedStorePath != dstPath)
                 state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path));
             state.allowAndSetStorePathString(dstPath, v);
diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc
index 63b05bdab..e071b4717 100644
--- a/src/libfetchers/cache.cc
+++ b/src/libfetchers/cache.cc
@@ -106,7 +106,7 @@ struct CacheImpl : Cache
     }
 
     void add(
-        ref<Store> store,
+        Store & store,
         const Attrs & inAttrs,
         const Attrs & infoAttrs,
         const StorePath & storePath,
@@ -115,13 +115,13 @@ struct CacheImpl : Cache
         _state.lock()->add.use()
             (attrsToJSON(inAttrs).dump())
             (attrsToJSON(infoAttrs).dump())
-            (store->printStorePath(storePath))
+            (store.printStorePath(storePath))
             (locked)
             (time(0)).exec();
     }
 
     std::optional<std::pair<Attrs, StorePath>> lookup(
-        ref<Store> store,
+        Store & store,
         const Attrs & inAttrs) override
     {
         if (auto res = lookupExpired(store, inAttrs)) {
@@ -134,7 +134,7 @@ struct CacheImpl : Cache
     }
 
     std::optional<Result> lookupExpired(
-        ref<Store> store,
+        Store & store,
         const Attrs & inAttrs) override
     {
         auto state(_state.lock());
@@ -148,19 +148,19 @@ struct CacheImpl : Cache
         }
 
         auto infoJSON = stmt.getStr(0);
-        auto storePath = store->parseStorePath(stmt.getStr(1));
+        auto storePath = store.parseStorePath(stmt.getStr(1));
         auto locked = stmt.getInt(2) != 0;
         auto timestamp = stmt.getInt(3);
 
-        store->addTempRoot(storePath);
-        if (!store->isValidPath(storePath)) {
+        store.addTempRoot(storePath);
+        if (!store.isValidPath(storePath)) {
             // FIXME: we could try to substitute 'storePath'.
             debug("ignoring disappeared cache entry '%s'", inAttrsJSON);
             return {};
         }
 
         debug("using cache entry '%s' -> '%s', '%s'",
-            inAttrsJSON, infoJSON, store->printStorePath(storePath));
+            inAttrsJSON, infoJSON, store.printStorePath(storePath));
 
         return Result {
             .expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),
diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh
index f70589267..791d77025 100644
--- a/src/libfetchers/cache.hh
+++ b/src/libfetchers/cache.hh
@@ -50,14 +50,14 @@ struct Cache
 
     /* Old cache for things that have a store path. */
     virtual void add(
-        ref<Store> store,
+        Store & store,
         const Attrs & inAttrs,
         const Attrs & infoAttrs,
         const StorePath & storePath,
         bool locked) = 0;
 
     virtual std::optional<std::pair<Attrs, StorePath>> lookup(
-        ref<Store> store,
+        Store & store,
         const Attrs & inAttrs) = 0;
 
     struct Result
@@ -68,7 +68,7 @@ struct Cache
     };
 
     virtual std::optional<Result> lookupExpired(
-        ref<Store> store,
+        Store & store,
         const Attrs & inAttrs) = 0;
 };
 
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 7ec1f9802..f309e5993 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -374,7 +374,7 @@ void InputScheme::clone(const Input & input, const Path & destDir) const
 std::pair<StorePath, Input> InputScheme::fetch(ref<Store> store, const Input & input)
 {
     auto [accessor, input2] = getAccessor(store, input);
-    auto storePath = SourcePath(accessor).fetchToStore(store, input2.getName());
+    auto storePath = SourcePath(accessor).fetchToStore(*store, input2.getName());
     return {storePath, input2};
 }
 
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 5dac66930..01cd28427 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -368,14 +368,14 @@ struct GitInputScheme : InputScheme
 
     RepoInfo getRepoInfo(const Input & input) const
     {
-        auto checkHashType = [&](const std::optional<Hash> & hash)
+        auto checkHashAlgorithm = [&](const std::optional<Hash> & hash)
         {
             if (hash.has_value() && !(hash->algo == HashAlgorithm::SHA1 || hash->algo == HashAlgorithm::SHA256))
                 throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true));
         };
 
         if (auto rev = input.getRev())
-            checkHashType(rev);
+            checkHashAlgorithm(rev);
 
         RepoInfo repoInfo;
 
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 70acb9354..498e41357 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -201,7 +201,7 @@ struct GitArchiveInputScheme : InputScheme
             {"rev", rev->gitRev()},
         });
 
-        if (auto res = getCache()->lookup(store, lockedAttrs)) {
+        if (auto res = getCache()->lookup(*store, lockedAttrs)) {
             input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified"));
             return {std::move(res->second), input};
         }
@@ -213,7 +213,7 @@ struct GitArchiveInputScheme : InputScheme
         input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
 
         getCache()->add(
-            store,
+            *store,
             lockedAttrs,
             {
                 {"rev", rev->gitRev()},
diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc
index 1f793bf1d..a647f5915 100644
--- a/src/libfetchers/input-accessor.cc
+++ b/src/libfetchers/input-accessor.cc
@@ -5,10 +5,10 @@
 namespace nix {
 
 StorePath InputAccessor::fetchToStore(
-    ref<Store> store,
+    Store & store,
     const CanonPath & path,
     std::string_view name,
-    FileIngestionMethod method,
+    ContentAddressMethod method,
     PathFilter * filter,
     RepairFlag repair)
 {
@@ -20,10 +20,24 @@ StorePath InputAccessor::fetchToStore(
     if (!filter && fingerprint) {
         cacheKey = fetchers::Attrs{
             {"_what", "fetchToStore"},
-            {"store", store->storeDir},
+            {"store", store.storeDir},
             {"name", std::string(name)},
             {"fingerprint", *fingerprint},
-            {"method", (uint8_t) method},
+            {
+                "method",
+                std::visit(overloaded {
+                    [](const TextIngestionMethod &) {
+                        return "text";
+                    },
+                    [](const FileIngestionMethod & fim) {
+                        switch (fim) {
+                        case FileIngestionMethod::Flat: return "flat";
+                        case FileIngestionMethod::Recursive: return "nar";
+                        default: assert(false);
+                        }
+                    },
+                }, method.raw),
+            },
             {"path", path.abs()}
         };
         if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) {
@@ -35,17 +49,14 @@ StorePath InputAccessor::fetchToStore(
 
     Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path)));
 
-    auto source = sinkToSource([&](Sink & sink) {
-        if (method == FileIngestionMethod::Recursive)
-            dumpPath(path, sink, filter ? *filter : defaultPathFilter);
-        else
-            readFile(path, sink);
-    });
+    auto filter2 = filter ? *filter : defaultPathFilter;
 
     auto storePath =
         settings.readOnlyMode
-        ? store->computeStorePathFromDump(*source, name, method, HashAlgorithm::SHA256).first
-        : store->addToStoreFromDump(*source, name, method, HashAlgorithm::SHA256, repair);
+        ? store.computeStorePath(
+            name, *this, path, method, HashAlgorithm::SHA256, {}, filter2).first
+        : store.addToStore(
+            name, *this, path, method, HashAlgorithm::SHA256, {}, filter2, repair);
 
     if (cacheKey)
         fetchers::getCache()->add(store, *cacheKey, {}, storePath, true);
@@ -60,9 +71,9 @@ std::ostream & operator << (std::ostream & str, const SourcePath & path)
 }
 
 StorePath SourcePath::fetchToStore(
-    ref<Store> store,
+    Store & store,
     std::string_view name,
-    FileIngestionMethod method,
+    ContentAddressMethod method,
     PathFilter * filter,
     RepairFlag repair) const
 {
diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh
index f385e6231..d2a21cb4b 100644
--- a/src/libfetchers/input-accessor.hh
+++ b/src/libfetchers/input-accessor.hh
@@ -30,10 +30,10 @@ struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this<Inpu
     }
 
     StorePath fetchToStore(
-        ref<Store> store,
+        Store & store,
         const CanonPath & path,
         std::string_view name = "source",
-        FileIngestionMethod method = FileIngestionMethod::Recursive,
+        ContentAddressMethod method = FileIngestionMethod::Recursive,
         PathFilter * filter = nullptr,
         RepairFlag repair = NoRepair);
 };
@@ -116,9 +116,9 @@ struct SourcePath
      * Copy this `SourcePath` to the Nix store.
      */
     StorePath fetchToStore(
-        ref<Store> store,
+        Store & store,
         std::string_view name = "source",
-        FileIngestionMethod method = FileIngestionMethod::Recursive,
+        ContentAddressMethod method = FileIngestionMethod::Recursive,
         PathFilter * filter = nullptr,
         RepairFlag repair = NoRepair) const;
 
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index 6056b9a3c..9982389ab 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -6,6 +6,7 @@
 #include "tarfile.hh"
 #include "store-api.hh"
 #include "url-parts.hh"
+#include "posix-source-accessor.hh"
 
 #include "fetch-settings.hh"
 
@@ -210,7 +211,12 @@ struct MercurialInputScheme : InputScheme
                     return files.count(file);
                 };
 
-                auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, filter);
+                PosixSourceAccessor accessor;
+                auto storePath = store->addToStore(
+                    input.getName(),
+                    accessor, CanonPath { actualPath },
+                    FileIngestionMethod::Recursive, HashAlgorithm::SHA256, {},
+                    filter);
 
                 return {std::move(storePath), input};
             }
@@ -218,7 +224,7 @@ struct MercurialInputScheme : InputScheme
 
         if (!input.getRef()) input.attrs.insert_or_assign("ref", "default");
 
-        auto checkHashType = [&](const std::optional<Hash> & hash)
+        auto checkHashAlgorithm = [&](const std::optional<Hash> & hash)
         {
             if (hash.has_value() && hash->algo != HashAlgorithm::SHA1)
                 throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(HashFormat::Base16, true));
@@ -227,7 +233,7 @@ struct MercurialInputScheme : InputScheme
 
         auto getLockedAttrs = [&]()
         {
-            checkHashType(input.getRev());
+            checkHashAlgorithm(input.getRev());
 
             return Attrs({
                 {"type", "hg"},
@@ -246,7 +252,7 @@ struct MercurialInputScheme : InputScheme
         };
 
         if (input.getRev()) {
-            if (auto res = getCache()->lookup(store, getLockedAttrs()))
+            if (auto res = getCache()->lookup(*store, getLockedAttrs()))
                 return makeResult(res->first, std::move(res->second));
         }
 
@@ -259,7 +265,7 @@ struct MercurialInputScheme : InputScheme
             {"ref", *input.getRef()},
         });
 
-        if (auto res = getCache()->lookup(store, unlockedAttrs)) {
+        if (auto res = getCache()->lookup(*store, unlockedAttrs)) {
             auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), HashAlgorithm::SHA1);
             if (!input.getRev() || input.getRev() == rev2) {
                 input.attrs.insert_or_assign("rev", rev2.gitRev());
@@ -305,7 +311,7 @@ struct MercurialInputScheme : InputScheme
         auto revCount = std::stoull(tokens[1]);
         input.attrs.insert_or_assign("ref", tokens[2]);
 
-        if (auto res = getCache()->lookup(store, getLockedAttrs()))
+        if (auto res = getCache()->lookup(*store, getLockedAttrs()))
             return makeResult(res->first, std::move(res->second));
 
         Path tmpDir = createTempDir();
@@ -315,7 +321,8 @@ struct MercurialInputScheme : InputScheme
 
         deletePath(tmpDir + "/.hg_archival.txt");
 
-        auto storePath = store->addToStore(name, tmpDir);
+        PosixSourceAccessor accessor;
+        auto storePath = store->addToStore(name, accessor, CanonPath { tmpDir });
 
         Attrs infoAttrs({
             {"rev", input.getRev()->gitRev()},
@@ -324,14 +331,14 @@ struct MercurialInputScheme : InputScheme
 
         if (!_input.getRev())
             getCache()->add(
-                store,
+                *store,
                 unlockedAttrs,
                 infoAttrs,
                 storePath,
                 false);
 
         getCache()->add(
-            store,
+            *store,
             getLockedAttrs(),
             infoAttrs,
             storePath,
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index 086366180..3b7709440 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -8,6 +8,7 @@
 #include "tarfile.hh"
 #include "types.hh"
 #include "split.hh"
+#include "posix-source-accessor.hh"
 
 namespace nix::fetchers {
 
@@ -26,7 +27,7 @@ DownloadFileResult downloadFile(
         {"name", name},
     });
 
-    auto cached = getCache()->lookupExpired(store, inAttrs);
+    auto cached = getCache()->lookupExpired(*store, inAttrs);
 
     auto useCached = [&]() -> DownloadFileResult
     {
@@ -91,7 +92,7 @@ DownloadFileResult downloadFile(
     }
 
     getCache()->add(
-        store,
+        *store,
         inAttrs,
         infoAttrs,
         *storePath,
@@ -99,7 +100,7 @@ DownloadFileResult downloadFile(
 
     if (url != res.effectiveUri)
         getCache()->add(
-            store,
+            *store,
             {
                 {"type", "file"},
                 {"url", res.effectiveUri},
@@ -130,7 +131,7 @@ DownloadTarballResult downloadTarball(
         {"name", name},
     });
 
-    auto cached = getCache()->lookupExpired(store, inAttrs);
+    auto cached = getCache()->lookupExpired(*store, inAttrs);
 
     if (cached && !cached->expired)
         return {
@@ -156,7 +157,8 @@ DownloadTarballResult downloadTarball(
             throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
         auto topDir = tmpDir + "/" + members.begin()->name;
         lastModified = lstat(topDir).st_mtime;
-        unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, defaultPathFilter, NoRepair);
+        PosixSourceAccessor accessor;
+        unpackedStorePath = store->addToStore(name, accessor, CanonPath { topDir }, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, {}, defaultPathFilter, NoRepair);
     }
 
     Attrs infoAttrs({
@@ -168,7 +170,7 @@ DownloadTarballResult downloadTarball(
         infoAttrs.emplace("immutableUrl", *res.immutableUrl);
 
     getCache()->add(
-        store,
+        *store,
         inAttrs,
         infoAttrs,
         *unpackedStorePath,
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 2837e8934..19aa283fc 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -300,8 +300,13 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
     }});
 }
 
-StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view name,
-                                               FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references)
+StorePath BinaryCacheStore::addToStoreFromDump(
+    Source & dump,
+    std::string_view name,
+    ContentAddressMethod method,
+    HashAlgorithm hashAlgo,
+    const StorePathSet & references,
+    RepairFlag repair)
 {
     if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256)
         unsupported("addToStoreFromDump");
@@ -309,15 +314,14 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n
         ValidPathInfo info {
             *this,
             name,
-            FixedOutputInfo {
-                .method = method,
-                .hash = nar.first,
-                .references = {
+            ContentAddressWithReferences::fromParts(
+                method,
+                nar.first,
+                {
                     .others = references,
                     // caller is not capable of creating a self-reference, because this is content-addressed without modulus
                     .self = false,
-                },
-            },
+                }),
             nar.first,
         };
         info.narSize = nar.second;
@@ -399,42 +403,36 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
 }
 
 StorePath BinaryCacheStore::addToStore(
-        std::string_view name,
-        const Path & srcPath,
-        FileIngestionMethod method,
-        HashAlgorithm hashAlgo,
-        PathFilter & filter,
-        RepairFlag repair,
-        const StorePathSet & references)
+    std::string_view name,
+    SourceAccessor & accessor,
+    const CanonPath & path,
+    ContentAddressMethod method,
+    HashAlgorithm hashAlgo,
+    const StorePathSet & references,
+    PathFilter & filter,
+    RepairFlag repair)
 {
     /* FIXME: Make BinaryCacheStore::addToStoreCommon support
        non-recursive+sha256 so we can just use the default
        implementation of this method in terms of addToStoreFromDump. */
 
-    HashSink sink { hashAlgo };
-    if (method == FileIngestionMethod::Recursive) {
-        dumpPath(srcPath, sink, filter);
-    } else {
-        readFile(srcPath, sink);
-    }
-    auto h = sink.finish().first;
+    auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter).first;
 
     auto source = sinkToSource([&](Sink & sink) {
-        dumpPath(srcPath, sink, filter);
+        accessor.dumpPath(path, sink, filter);
     });
     return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info {
             *this,
             name,
-            FixedOutputInfo {
-                .method = method,
-                .hash = h,
-                .references = {
+            ContentAddressWithReferences::fromParts(
+                method,
+                h,
+                {
                     .others = references,
                     // caller is not capable of creating a self-reference, because this is content-addressed without modulus
                     .self = false,
-                },
-            },
+                }),
             nar.first,
         };
         info.narSize = nar.second;
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 395e1b479..dbe4ac180 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -123,17 +123,23 @@ public:
     void addToStore(const ValidPathInfo & info, Source & narSource,
         RepairFlag repair, CheckSigsFlag checkSigs) override;
 
-    StorePath addToStoreFromDump(Source & dump, std::string_view name,
-                                 FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) override;
+    StorePath addToStoreFromDump(
+        Source & dump,
+        std::string_view name,
+        ContentAddressMethod method,
+        HashAlgorithm hashAlgo,
+        const StorePathSet & references,
+        RepairFlag repair) override;
 
     StorePath addToStore(
-            std::string_view name,
-            const Path & srcPath,
-            FileIngestionMethod method,
-            HashAlgorithm hashAlgo,
-            PathFilter & filter,
-            RepairFlag repair,
-            const StorePathSet & references) override;
+        std::string_view name,
+        SourceAccessor & accessor,
+        const CanonPath & srcPath,
+        ContentAddressMethod method,
+        HashAlgorithm hashAlgo,
+        const StorePathSet & references,
+        PathFilter & filter,
+        RepairFlag repair) override;
 
     StorePath addTextToStore(
         std::string_view name,
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 802b39f84..e4828dd2f 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -20,6 +20,7 @@
 #include "child.hh"
 #include "unix-domain-socket.hh"
 #include "posix-fs-canonicalise.hh"
+#include "posix-source-accessor.hh"
 
 #include <regex>
 #include <queue>
@@ -1290,13 +1291,14 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In
     { throw Error("queryPathFromHashPart"); }
 
     StorePath addToStore(
-            std::string_view name,
-            const Path & srcPath,
-            FileIngestionMethod method,
-            HashAlgorithm hashAlgo,
-            PathFilter & filter,
-            RepairFlag repair,
-            const StorePathSet & references) override
+        std::string_view name,
+        SourceAccessor & accessor,
+        const CanonPath & srcPath,
+        ContentAddressMethod method,
+        HashAlgorithm hashAlgo,
+        const StorePathSet & references,
+        PathFilter & filter,
+        RepairFlag repair) override
     { throw Error("addToStore"); }
 
     void addToStore(const ValidPathInfo & info, Source & narSource,
@@ -1318,14 +1320,14 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In
     }
 
     StorePath addToStoreFromDump(
-            Source & dump,
-            std::string_view name,
-            FileIngestionMethod method,
-            HashAlgorithm hashAlgo,
-            RepairFlag repair,
-            const StorePathSet & references) override
+        Source & dump,
+        std::string_view name,
+        ContentAddressMethod method,
+        HashAlgorithm hashAlgo,
+        const StorePathSet & references,
+        RepairFlag repair) override
     {
-        auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references);
+        auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, references, repair);
         goal.addDependency(path);
         return path;
     }
@@ -2453,8 +2455,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                 throw BuildError(
                     "output path %1% without valid stats info",
                     actualPath);
-            if (outputHash.method == ContentAddressMethod { FileIngestionMethod::Flat } ||
-                outputHash.method == ContentAddressMethod { TextIngestionMethod {} })
+            if (outputHash.method.getFileIngestionMethod() == FileIngestionMethod::Flat)
             {
                 /* The output path should be a regular file without execute permission. */
                 if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0)
@@ -2466,38 +2467,23 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
             rewriteOutput(outputRewrites);
             /* FIXME optimize and deduplicate with addToStore */
             std::string oldHashPart { scratchPath->hashPart() };
-            HashModuloSink caSink {outputHash.hashAlgo, oldHashPart };
-            std::visit(overloaded {
-                [&](const TextIngestionMethod &) {
-                    readFile(actualPath, caSink);
-                },
-                [&](const FileIngestionMethod & m2) {
-                    switch (m2) {
-                    case FileIngestionMethod::Recursive:
-                        dumpPath(actualPath, caSink);
-                        break;
-                    case FileIngestionMethod::Flat:
-                        readFile(actualPath, caSink);
-                        break;
-                    }
-                },
-            }, outputHash.method.raw);
-            auto got = caSink.finish().first;
+            auto got = ({
+                HashModuloSink caSink { outputHash.hashAlgo, oldHashPart };
+                PosixSourceAccessor accessor;
+                dumpPath(
+                    accessor, CanonPath { actualPath },
+                    caSink,
+                    outputHash.method.getFileIngestionMethod());
+                caSink.finish().first;
+            });
 
-            auto optCA = ContentAddressWithReferences::fromPartsOpt(
-                outputHash.method,
-                std::move(got),
-                rewriteRefs());
-            if (!optCA) {
-                // TODO track distinct failure modes separately (at the time of
-                // writing there is just one but `nullopt` is unclear) so this
-                // message can't get out of sync.
-                throw BuildError("output path '%s' has illegal content address, probably a spurious self-reference with text hashing");
-            }
             ValidPathInfo newInfo0 {
                 worker.store,
                 outputPathName(drv->name, outputName),
-                std::move(*optCA),
+                ContentAddressWithReferences::fromParts(
+                    outputHash.method,
+                    std::move(got),
+                    rewriteRefs()),
                 Hash::dummy,
             };
             if (*scratchPath != newInfo0.path) {
@@ -2511,9 +2497,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                                std::string(newInfo0.path.hashPart())}});
             }
 
-            HashResult narHashAndSize = hashPath(HashAlgorithm::SHA256, actualPath);
-            newInfo0.narHash = narHashAndSize.first;
-            newInfo0.narSize = narHashAndSize.second;
+            {
+                PosixSourceAccessor accessor;
+                HashResult narHashAndSize = hashPath(
+                    accessor, CanonPath { actualPath },
+                    FileIngestionMethod::Recursive, HashAlgorithm::SHA256);
+                newInfo0.narHash = narHashAndSize.first;
+                newInfo0.narSize = narHashAndSize.second;
+            }
 
             assert(newInfo0.ca);
             return newInfo0;
@@ -2531,7 +2522,10 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                         std::string { scratchPath->hashPart() },
                         std::string { requiredFinalPath.hashPart() });
                 rewriteOutput(outputRewrites);
-                auto narHashAndSize = hashPath(HashAlgorithm::SHA256, actualPath);
+                PosixSourceAccessor accessor;
+                HashResult narHashAndSize = hashPath(
+                    accessor, CanonPath { actualPath },
+                    FileIngestionMethod::Recursive, HashAlgorithm::SHA256);
                 ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
                 newInfo0.narSize = narHashAndSize.second;
                 auto refs = rewriteRefs();
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 9b8c36286..399ad47fd 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -519,7 +519,9 @@ bool Worker::pathContentsGood(const StorePath & path)
     if (!pathExists(store.printStorePath(path)))
         res = false;
     else {
-        HashResult current = hashPath(info->narHash.algo, store.printStorePath(path));
+        HashResult current = hashPath(
+            *store.getFSAccessor(), CanonPath { store.printStorePath(path) },
+            FileIngestionMethod::Recursive, info->narHash.algo);
         Hash nullHash(HashAlgorithm::SHA256);
         res = info->narHash == nullHash || info->narHash == current.first;
     }
diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc
index f42a13126..fc408f5af 100644
--- a/src/libstore/content-address.cc
+++ b/src/libstore/content-address.cc
@@ -50,6 +50,18 @@ std::string ContentAddressMethod::render(HashAlgorithm ha) const
     }, raw);
 }
 
+FileIngestionMethod ContentAddressMethod::getFileIngestionMethod() const
+{
+    return std::visit(overloaded {
+        [&](const TextIngestionMethod & th) {
+            return FileIngestionMethod::Flat;
+        },
+        [&](const FileIngestionMethod & fim) {
+            return fim;
+        }
+    }, raw);
+}
+
 std::string ContentAddress::render() const
 {
     return std::visit(overloaded {
@@ -79,7 +91,7 @@ static std::pair<ContentAddressMethod, HashAlgorithm> parseContentAddressMethodP
         prefix = *optPrefix;
     }
 
-    auto parseHashType_ = [&](){
+    auto parseHashAlgorithm_ = [&](){
         auto hashTypeRaw = splitPrefixTo(rest, ':');
         if (!hashTypeRaw)
             throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", wholeInput);
@@ -90,7 +102,7 @@ static std::pair<ContentAddressMethod, HashAlgorithm> parseContentAddressMethodP
     // Switch on prefix
     if (prefix == "text") {
         // No parsing of the ingestion method, "text" only support flat.
-        HashAlgorithm hashAlgo = parseHashType_();
+        HashAlgorithm hashAlgo = parseHashAlgorithm_();
         return {
             TextIngestionMethod {},
             std::move(hashAlgo),
@@ -100,7 +112,7 @@ static std::pair<ContentAddressMethod, HashAlgorithm> parseContentAddressMethodP
         auto method = FileIngestionMethod::Flat;
         if (splitPrefix(rest, "r:"))
             method = FileIngestionMethod::Recursive;
-        HashAlgorithm hashAlgo = parseHashType_();
+        HashAlgorithm hashAlgo = parseHashAlgorithm_();
         return {
             std::move(method),
             std::move(hashAlgo),
@@ -176,13 +188,13 @@ ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const Con
     }, ca.method.raw);
 }
 
-std::optional<ContentAddressWithReferences> ContentAddressWithReferences::fromPartsOpt(
-    ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept
+ContentAddressWithReferences ContentAddressWithReferences::fromParts(
+    ContentAddressMethod method, Hash hash, StoreReferences refs)
 {
     return std::visit(overloaded {
-        [&](TextIngestionMethod _) -> std::optional<ContentAddressWithReferences> {
+        [&](TextIngestionMethod _) -> ContentAddressWithReferences {
             if (refs.self)
-                return std::nullopt;
+                throw Error("self-reference not allowed with text hashing");
             return ContentAddressWithReferences {
                 TextInfo {
                     .hash = std::move(hash),
@@ -190,7 +202,7 @@ std::optional<ContentAddressWithReferences> ContentAddressWithReferences::fromPa
                 }
             };
         },
-        [&](FileIngestionMethod m2) -> std::optional<ContentAddressWithReferences> {
+        [&](FileIngestionMethod m2) -> ContentAddressWithReferences {
             return ContentAddressWithReferences {
                 FixedOutputInfo {
                     .method = m2,
diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh
index 05234da38..6863ad260 100644
--- a/src/libstore/content-address.hh
+++ b/src/libstore/content-address.hh
@@ -4,6 +4,7 @@
 #include <variant>
 #include "hash.hh"
 #include "path.hh"
+#include "file-content-address.hh"
 #include "comparator.hh"
 #include "variant-wrapper.hh"
 
@@ -31,22 +32,6 @@ namespace nix {
  */
 struct TextIngestionMethod : std::monostate { };
 
-/**
- * An enumeration of the main ways we can serialize file system
- * objects.
- */
-enum struct FileIngestionMethod : uint8_t {
-    /**
-     * Flat-file hashing. Directly ingest the contents of a single file
-     */
-    Flat = 0,
-    /**
-     * Recursive (or NAR) hashing. Serializes the file-system object in Nix
-     * Archive format and ingest that
-     */
-    Recursive = 1
-};
-
 /**
  * Compute the prefix to the hash algorithm which indicates how the
  * files were ingested.
@@ -54,7 +39,7 @@ enum struct FileIngestionMethod : uint8_t {
 std::string makeFileIngestionPrefix(FileIngestionMethod m);
 
 /**
- * An enumeration of all the ways we can serialize file system objects.
+ * An enumeration of all the ways we can content-address store objects.
  *
  * Just the type of a content address. Combine with the hash itself, and
  * we have a `ContentAddress` as defined below. Combine that, in turn,
@@ -102,7 +87,15 @@ struct ContentAddressMethod
      *
      * The rough inverse of `parse()`.
      */
-    std::string render(HashAlgorithm ha) const;
+    std::string render(HashAlgorithm ht) const;
+
+    /**
+     * Get the underlying way to content-address file system objects.
+     *
+     * Different ways of hashing store objects may use the same method
+     * for hashing file systeme objects.
+     */
+    FileIngestionMethod getFileIngestionMethod() const;
 };
 
 
@@ -266,11 +259,12 @@ struct ContentAddressWithReferences
      *
      * @param refs References to other store objects or oneself.
      *
-     * Do note that not all combinations are supported; `nullopt` is
-     * returns for invalid combinations.
+     * @note note that all combinations are supported. This is a
+     * *partial function* and exceptions will be thrown for invalid
+     * combinations.
      */
-    static std::optional<ContentAddressWithReferences> fromPartsOpt(
-        ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept;
+    static ContentAddressWithReferences fromParts(
+        ContentAddressMethod method, Hash hash, StoreReferences refs);
 
     ContentAddressMethod getMethod() const;
 
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index a112d6d31..574263c68 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -403,22 +403,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                 auto [contentAddressMethod, hashAlgo_] = ContentAddressMethod::parse(camStr);
                 auto hashAlgo = hashAlgo_; // work around clang bug
                 FramedSource source(from);
-                // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
-                return std::visit(overloaded {
-                    [&](const TextIngestionMethod &) {
-                        if (hashAlgo != HashAlgorithm::SHA256)
-                            throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given",
-                                name, printHashAlgo(hashAlgo));
-                        // We could stream this by changing Store
-                        std::string contents = source.drain();
-                        auto path = store->addTextToStore(name, contents, refs, repair);
-                        return store->queryPathInfo(path);
-                    },
-                    [&](const FileIngestionMethod & fim) {
-                        auto path = store->addToStoreFromDump(source, name, fim, hashAlgo, repair, refs);
-                        return store->queryPathInfo(path);
-                    },
-                }, contentAddressMethod.raw);
+                // TODO these two steps are essentially RemoteStore::addCAToStore. Move it up to Store.
+                auto path = store->addToStoreFromDump(source, name, contentAddressMethod, hashAlgo, refs, repair);
+                return store->queryPathInfo(path);
             }();
             logger->stopWork();
 
diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh
index c40c256bb..8b142ba2a 100644
--- a/src/libstore/legacy-ssh-store.hh
+++ b/src/libstore/legacy-ssh-store.hh
@@ -59,13 +59,14 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
     { unsupported("queryPathFromHashPart"); }
 
     StorePath addToStore(
-            std::string_view name,
-            const Path & srcPath,
-            FileIngestionMethod method,
-            HashAlgorithm hashAlgo,
-            PathFilter & filter,
-            RepairFlag repair,
-            const StorePathSet & references) override
+        std::string_view name,
+        SourceAccessor & accessor,
+        const CanonPath & srcPath,
+        ContentAddressMethod method,
+        HashAlgorithm hashAlgo,
+        const StorePathSet & references,
+        PathFilter & filter,
+        RepairFlag repair) override
     { unsupported("addToStore"); }
 
     StorePath addTextToStore(
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 7e82bae28..cd8bf24f8 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -13,6 +13,7 @@
 #include "compression.hh"
 #include "signals.hh"
 #include "posix-fs-canonicalise.hh"
+#include "posix-source-accessor.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -1088,11 +1089,22 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             if (info.ca) {
                 auto & specified = *info.ca;
-                auto actualHash = hashCAPath(
-                    specified.method,
-                    specified.hash.algo,
-                    info.path
-                );
+                auto actualHash = ({
+                    HashModuloSink caSink {
+                        specified.hash.algo,
+                        std::string { info.path.hashPart() },
+                    };
+                    PosixSourceAccessor accessor;
+                    dumpPath(
+                        *getFSAccessor(false),
+                        CanonPath { printStorePath(info.path) },
+                        caSink,
+                        specified.method.getFileIngestionMethod());
+                    ContentAddress {
+                        .method = specified.method,
+                        .hash = caSink.finish().first,
+                    };
+                });
                 if (specified.hash != actualHash.hash) {
                     throw Error("ca hash mismatch importing path '%s';\n  specified: %s\n  got:       %s",
                         printStorePath(info.path),
@@ -1115,8 +1127,13 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 }
 
 
-StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name,
-                                         FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references)
+StorePath LocalStore::addToStoreFromDump(
+    Source & source0,
+    std::string_view name,
+    ContentAddressMethod method,
+    HashAlgorithm hashAlgo,
+    const StorePathSet & references,
+    RepairFlag repair)
 {
     /* For computing the store path. */
     auto hashSink = std::make_unique<HashSink>(hashAlgo);
@@ -1166,25 +1183,21 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
         delTempDir = std::make_unique<AutoDelete>(tempDir);
         tempPath = tempDir + "/x";
 
-        if (method == FileIngestionMethod::Recursive)
-            restorePath(tempPath, bothSource);
-        else
-            writeFile(tempPath, bothSource);
+        restorePath(tempPath, bothSource, method.getFileIngestionMethod());
 
         dump.clear();
     }
 
     auto [hash, size] = hashSink->finish();
 
-    ContentAddressWithReferences desc = FixedOutputInfo {
-        .method = method,
-        .hash = hash,
-        .references = {
+    auto desc = ContentAddressWithReferences::fromParts(
+        method,
+        hash,
+        {
             .others = references,
             // caller is not capable of creating a self-reference, because this is content-addressed without modulus
             .self = false,
-        },
-    };
+        });
 
     auto dstPath = makeFixedOutputPathFromCA(name, desc);
 
@@ -1207,11 +1220,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
 
             if (inMemory) {
                 StringSource dumpSource { dump };
-                /* Restore from the NAR in memory. */
-                if (method == FileIngestionMethod::Recursive)
-                    restorePath(realPath, dumpSource);
-                else
-                    writeFile(realPath, dumpSource);
+                /* Restore from the buffer in memory. */
+                restorePath(realPath, dumpSource, method.getFileIngestionMethod());
             } else {
                 /* Move the temporary path we restored above. */
                 moveFile(tempPath, realPath);
@@ -1389,7 +1399,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
         for (auto & link : readDirectory(linksDir)) {
             printMsg(lvlTalkative, "checking contents of '%s'", link.name);
             Path linkPath = linksDir + "/" + link.name;
-            std::string hash = hashPath(HashAlgorithm::SHA256, linkPath).first.to_string(HashFormat::Nix32, false);
+            PosixSourceAccessor accessor;
+            std::string hash = hashPath(
+                accessor, CanonPath { linkPath },
+                FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first.to_string(HashFormat::Nix32, false);
             if (hash != link.name) {
                 printError("link '%s' was modified! expected hash '%s', got '%s'",
                     linkPath, link.name, hash);
@@ -1696,42 +1709,6 @@ void LocalStore::queryRealisationUncached(const DrvOutput & id,
     }
 }
 
-ContentAddress LocalStore::hashCAPath(
-    const ContentAddressMethod & method, const HashAlgorithm & hashAlgo,
-    const StorePath & path)
-{
-    return hashCAPath(method, hashAlgo, Store::toRealPath(path), path.hashPart());
-}
-
-ContentAddress LocalStore::hashCAPath(
-    const ContentAddressMethod & method,
-    const HashAlgorithm & hashAlgo,
-    const Path & path,
-    const std::string_view pathHash
-)
-{
-    HashModuloSink caSink ( hashAlgo, std::string(pathHash) );
-    std::visit(overloaded {
-        [&](const TextIngestionMethod &) {
-            readFile(path, caSink);
-        },
-        [&](const FileIngestionMethod & m2) {
-            switch (m2) {
-            case FileIngestionMethod::Recursive:
-                dumpPath(path, caSink);
-                break;
-            case FileIngestionMethod::Flat:
-                readFile(path, caSink);
-                break;
-            }
-        },
-    }, method.raw);
-    return ContentAddress {
-        .method = method,
-        .hash = caSink.finish().first,
-    };
-}
-
 void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log)
 {
     assert(drvPath.isDerivation());
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index ee605b5a2..a8323fe5a 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -177,8 +177,13 @@ public:
     void addToStore(const ValidPathInfo & info, Source & source,
         RepairFlag repair, CheckSigsFlag checkSigs) override;
 
-    StorePath addToStoreFromDump(Source & dump, std::string_view name,
-                                 FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) override;
+    StorePath addToStoreFromDump(
+        Source & dump,
+        std::string_view name,
+        ContentAddressMethod method,
+        HashAlgorithm hashAlgo,
+        const StorePathSet & references,
+        RepairFlag repair) override;
 
     StorePath addTextToStore(
         std::string_view name,
@@ -350,19 +355,6 @@ private:
     void signPathInfo(ValidPathInfo & info);
     void signRealisation(Realisation &);
 
-    // XXX: Make a generic `Store` method
-    ContentAddress hashCAPath(
-        const ContentAddressMethod & method,
-        const HashAlgorithm & hashAlgo,
-        const StorePath & path);
-
-    ContentAddress hashCAPath(
-        const ContentAddressMethod & method,
-        const HashAlgorithm & hashAlgo,
-        const Path & path,
-        const std::string_view pathHash
-    );
-
     void addBuildLog(const StorePath & drvPath, std::string_view log) override;
 
     friend struct LocalDerivationGoal;
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index b395453d1..a494e6ecc 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -2,6 +2,7 @@
 #include "globals.hh"
 #include "signals.hh"
 #include "posix-fs-canonicalise.hh"
+#include "posix-source-accessor.hh"
 
 #include <cstdlib>
 #include <cstring>
@@ -146,7 +147,12 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
        Also note that if `path' is a symlink, then we're hashing the
        contents of the symlink (i.e. the result of readlink()), not
        the contents of the target (which may not even exist). */
-    Hash hash = hashPath(HashAlgorithm::SHA256, path).first;
+    Hash hash = ({
+        PosixSourceAccessor accessor;
+        hashPath(
+            accessor, CanonPath { path },
+            FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first;
+    });
     debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true));
 
     /* Check if this is a known hash. */
@@ -156,7 +162,12 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     if (pathExists(linkPath)) {
         auto stLink = lstat(linkPath);
         if (st.st_size != stLink.st_size
-            || (repair && hash != hashPath(HashAlgorithm::SHA256, linkPath).first))
+            || (repair && hash != ({
+                PosixSourceAccessor accessor;
+                hashPath(
+                    accessor, CanonPath { linkPath },
+                    FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first;
+           })))
         {
             // XXX: Consider overwriting linkPath with our valid version.
             warn("removing corrupted link '%s'", linkPath);
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index dd6347468..567776b67 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -502,8 +502,13 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
 }
 
 
-StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name,
-                                          FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references)
+StorePath RemoteStore::addToStoreFromDump(
+    Source & dump,
+    std::string_view name,
+    ContentAddressMethod method,
+    HashAlgorithm hashAlgo,
+    const StorePathSet & references,
+    RepairFlag repair)
 {
     return addCAToStore(dump, name, method, hashAlgo, references, repair)->path;
 }
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index f2e34c1a3..68824a737 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -82,10 +82,15 @@ public:
             RepairFlag repair);
 
     /**
-     * Add a content-addressable store path. Does not support references. `dump` will be drained.
+     * Add a content-addressable store path. `dump` will be drained.
      */
-    StorePath addToStoreFromDump(Source & dump, std::string_view name,
-                                 FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override;
+    StorePath addToStoreFromDump(
+        Source & dump,
+        std::string_view name,
+        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
+        const StorePathSet & references = StorePathSet(),
+        RepairFlag repair = NoRepair) override;
 
     void addToStore(const ValidPathInfo & info, Source & nar,
         RepairFlag repair, CheckSigsFlag checkSigs) override;
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 7f35e74af..5b4c6c765 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -232,22 +232,28 @@ StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const
 }
 
 
-std::pair<StorePath, Hash> StoreDirConfig::computeStorePathFromDump(
-        Source & dump,
-        std::string_view name,
-        FileIngestionMethod method,
-        HashAlgorithm hashAlgo,
-        const StorePathSet & references) const
+std::pair<StorePath, Hash> StoreDirConfig::computeStorePath(
+    std::string_view name,
+    SourceAccessor & accessor,
+    const CanonPath & path,
+    ContentAddressMethod method,
+    HashAlgorithm hashAlgo,
+    const StorePathSet & references,
+    PathFilter & filter) const
 {
-    HashSink sink(hashAlgo);
-    dump.drainInto(sink);
-    auto h = sink.finish().first;
-    FixedOutputInfo caInfo {
-        .method = method,
-        .hash = h,
-        .references = {},
+    auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter).first;
+    return {
+        makeFixedOutputPathFromCA(
+            name,
+            ContentAddressWithReferences::fromParts(
+                method,
+                h,
+                {
+                    .others = references,
+                    .self = false,
+                })),
+        h,
     };
-    return std::make_pair(makeFixedOutputPath(name, caInfo), h);
 }
 
 
@@ -264,22 +270,19 @@ StorePath StoreDirConfig::computeStorePathForText(
 
 
 StorePath Store::addToStore(
-        std::string_view name,
-        const Path & _srcPath,
-        FileIngestionMethod method,
-        HashAlgorithm hashAlgo,
-        PathFilter & filter,
-        RepairFlag repair,
-        const StorePathSet & references)
+    std::string_view name,
+    SourceAccessor & accessor,
+    const CanonPath & path,
+    ContentAddressMethod method,
+    HashAlgorithm hashAlgo,
+    const StorePathSet & references,
+    PathFilter & filter,
+    RepairFlag repair)
 {
-    Path srcPath(absPath(_srcPath));
     auto source = sinkToSource([&](Sink & sink) {
-        if (method == FileIngestionMethod::Recursive)
-            dumpPath(srcPath, sink, filter);
-        else
-            readFile(srcPath, sink);
+        dumpPath(accessor, path, sink, method.getFileIngestionMethod(), filter);
     });
-    return addToStoreFromDump(*source, name, method, hashAlgo, repair, references);
+    return addToStoreFromDump(*source, name, method, hashAlgo, references, repair);
 }
 
 void Store::addMultipleToStore(
@@ -404,9 +407,13 @@ digraph graphname {
     fileSink -> caHashSink
 }
 */
-ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
-                                    FileIngestionMethod method, HashAlgorithm hashAlgo,
-                                    std::optional<Hash> expectedCAHash)
+ValidPathInfo Store::addToStoreSlow(
+    std::string_view name,
+    SourceAccessor & accessor,
+    const CanonPath & srcPath,
+    ContentAddressMethod method, HashAlgorithm hashAlgo,
+    const StorePathSet & references,
+    std::optional<Hash> expectedCAHash)
 {
     HashSink narHashSink { HashAlgorithm::SHA256 };
     HashSink caHashSink { hashAlgo };
@@ -425,7 +432,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
        srcPath. The fact that we use scratchpadSink as a temporary buffer here
        is an implementation detail. */
     auto fileSource = sinkToSource([&](Sink & scratchpadSink) {
-        dumpPath(srcPath, scratchpadSink);
+        accessor.dumpPath(srcPath, scratchpadSink);
     });
 
     /* tapped provides the same data as fileSource, but we also write all the
@@ -433,9 +440,11 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
     TeeSource tapped { *fileSource, narSink };
 
     NullParseSink blank;
-    auto & parseSink = method == FileIngestionMethod::Flat
+    auto & parseSink = method.getFileIngestionMethod() == FileIngestionMethod::Flat
         ? (ParseSink &) fileSink
-        : (ParseSink &) blank;
+        : method.getFileIngestionMethod() == FileIngestionMethod::Recursive
+        ? (ParseSink &) blank
+        : (abort(), (ParseSink &)*(ParseSink *)nullptr); // handled both cases
 
     /* The information that flows from tapped (besides being replicated in
        narSink), is now put in parseSink. */
@@ -452,21 +461,24 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
     if (expectedCAHash && expectedCAHash != hash)
         throw Error("hash mismatch for '%s'", srcPath);
 
+
     ValidPathInfo info {
         *this,
         name,
-        FixedOutputInfo {
-            .method = method,
-            .hash = hash,
-            .references = {},
-        },
+        ContentAddressWithReferences::fromParts(
+            method,
+            hash,
+            {
+                .others = references,
+                .self = false,
+            }),
         narHash,
     };
     info.narSize = narSize;
 
     if (!isValidPath(info.path)) {
         auto source = sinkToSource([&](Sink & scratchpadSink) {
-            dumpPath(srcPath, scratchpadSink);
+            accessor.dumpPath(srcPath, scratchpadSink);
         });
         addToStore(info, *source);
     }
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 2c883ce97..fc0a82a73 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -427,22 +427,28 @@ public:
      * libutil/archive.hh).
      */
     virtual StorePath addToStore(
-            std::string_view name,
-            const Path & srcPath,
-            FileIngestionMethod method = FileIngestionMethod::Recursive,
-            HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
-            PathFilter & filter = defaultPathFilter,
-            RepairFlag repair = NoRepair,
-            const StorePathSet & references = StorePathSet());
+        std::string_view name,
+        SourceAccessor & accessor,
+        const CanonPath & path,
+        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
+        const StorePathSet & references = StorePathSet(),
+        PathFilter & filter = defaultPathFilter,
+        RepairFlag repair = NoRepair);
 
     /**
      * Copy the contents of a path to the store and register the
      * validity the resulting path, using a constant amount of
      * memory.
      */
-    ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath,
-                                 FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
-                                 std::optional<Hash> expectedCAHash = {});
+    ValidPathInfo addToStoreSlow(
+        std::string_view name,
+        SourceAccessor & accessor,
+        const CanonPath & path,
+        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
+        const StorePathSet & references = StorePathSet(),
+        std::optional<Hash> expectedCAHash = {});
 
     /**
      * Like addToStore(), but the contents of the path are contained
@@ -453,9 +459,13 @@ public:
      *
      * \todo remove?
      */
-    virtual StorePath addToStoreFromDump(Source & dump, std::string_view name,
-                                         FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, RepairFlag repair = NoRepair,
-                                         const StorePathSet & references = StorePathSet())
+    virtual StorePath addToStoreFromDump(
+        Source & dump,
+        std::string_view name,
+        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
+        const StorePathSet & references = StorePathSet(),
+        RepairFlag repair = NoRepair)
     { unsupported("addToStoreFromDump"); }
 
     /**
diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh
index 8dafca096..0fc8ded9c 100644
--- a/src/libstore/store-dir-config.hh
+++ b/src/libstore/store-dir-config.hh
@@ -91,15 +91,17 @@ struct StoreDirConfig : public Config
     StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const;
 
     /**
-     * Read-only variant of addToStoreFromDump(). It returns the store
-     * path to which a NAR or flat file would be written.
+     * Read-only variant of addToStore(). It returns the store
+     * path for the given file sytem object.
      */
-    std::pair<StorePath, Hash> computeStorePathFromDump(
-        Source & dump,
+    std::pair<StorePath, Hash> computeStorePath(
         std::string_view name,
-        FileIngestionMethod method = FileIngestionMethod::Recursive,
+        SourceAccessor & accessor,
+        const CanonPath & path,
+        ContentAddressMethod method = FileIngestionMethod::Recursive,
         HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
-        const StorePathSet & references = {}) const;
+        const StorePathSet & references = {},
+        PathFilter & filter = defaultPathFilter) const;
 
     /**
      * Preparatory part of addTextToStore().
diff --git a/src/libutil/file-content-address.cc b/src/libutil/file-content-address.cc
new file mode 100644
index 000000000..9917986f6
--- /dev/null
+++ b/src/libutil/file-content-address.cc
@@ -0,0 +1,49 @@
+#include "file-content-address.hh"
+#include "archive.hh"
+
+namespace nix {
+
+void dumpPath(
+    SourceAccessor & accessor, const CanonPath & path,
+    Sink & sink,
+    FileIngestionMethod method,
+    PathFilter & filter)
+{
+    switch (method) {
+    case FileIngestionMethod::Flat:
+        accessor.readFile(path, sink);
+        break;
+    case FileIngestionMethod::Recursive:
+        accessor.dumpPath(path, sink, filter);
+        break;
+    }
+}
+
+
+void restorePath(
+    const Path & path,
+    Source & source,
+    FileIngestionMethod method)
+{
+    switch (method) {
+    case FileIngestionMethod::Flat:
+        writeFile(path, source);
+        break;
+    case FileIngestionMethod::Recursive:
+        restorePath(path, source);
+        break;
+    }
+}
+
+
+HashResult hashPath(
+    SourceAccessor & accessor, const CanonPath & path,
+    FileIngestionMethod method, HashAlgorithm ht,
+    PathFilter & filter)
+{
+    HashSink sink { ht };
+    dumpPath(accessor, path, sink, method, filter);
+    return sink.finish();
+}
+
+}
diff --git a/src/libutil/file-content-address.hh b/src/libutil/file-content-address.hh
new file mode 100644
index 000000000..8e93f5847
--- /dev/null
+++ b/src/libutil/file-content-address.hh
@@ -0,0 +1,56 @@
+#pragma once
+///@file
+
+#include "source-accessor.hh"
+#include "fs-sink.hh"
+#include "util.hh"
+
+namespace nix {
+
+/**
+ * An enumeration of the main ways we can serialize file system
+ * objects.
+ */
+enum struct FileIngestionMethod : uint8_t {
+    /**
+     * Flat-file hashing. Directly ingest the contents of a single file
+     */
+    Flat = 0,
+    /**
+     * Recursive (or NAR) hashing. Serializes the file-system object in
+     * Nix Archive format and ingest that.
+     */
+    Recursive = 1,
+};
+
+/**
+ * Dump a serialization of the given file system object.
+ */
+void dumpPath(
+    SourceAccessor & accessor, const CanonPath & path,
+    Sink & sink,
+    FileIngestionMethod method,
+    PathFilter & filter = defaultPathFilter);
+
+/**
+ * Restore a serialization of the given file system object.
+ *
+ * @TODO use an arbitrary `ParseSink`.
+ */
+void restorePath(
+    const Path & path,
+    Source & source,
+    FileIngestionMethod method);
+
+/**
+ * Compute the hash of the given file system object according to the
+ * given method.
+ *
+ * The hash is defined as (essentially) hashString(ht, dumpPath(path)).
+ */
+HashResult hashPath(
+    SourceAccessor & accessor, const CanonPath & path,
+    FileIngestionMethod method, HashAlgorithm ht,
+    PathFilter & filter = defaultPathFilter);
+
+}
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 30456ae5c..502afbda2 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -367,15 +367,6 @@ HashResult HashSink::currentHash()
 }
 
 
-HashResult hashPath(
-        HashAlgorithm ha, const Path & path, PathFilter & filter)
-{
-    HashSink sink(ha);
-    dumpPath(path, sink, filter);
-    return sink.finish();
-}
-
-
 Hash compressHash(const Hash & hash, unsigned int newSize)
 {
     Hash h(hash.algo);
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index 7bed9e2bd..2fe9a53f5 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -168,14 +168,11 @@ Hash hashString(HashAlgorithm ha, std::string_view s);
 Hash hashFile(HashAlgorithm ha, const Path & path);
 
 /**
- * Compute the hash of the given path, serializing as a Nix Archive and
- * then hashing that.
+ * The final hash and the number of bytes digested.
  *
- * The hash is defined as (essentially) hashString(ht, dumpPath(path)).
+ * @todo Convert to proper struct
  */
 typedef std::pair<Hash, uint64_t> HashResult;
-HashResult hashPath(HashAlgorithm ha, const Path & path,
-                    PathFilter & filter = defaultPathFilter);
 
 /**
  * Compress a hash to the specified number of bytes by cyclically
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index d361dc0ac..0a0a3ab1a 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -13,6 +13,7 @@
 #include "shared.hh"
 #include "graphml.hh"
 #include "legacy.hh"
+#include "posix-source-accessor.hh"
 #include "path-with-outputs.hh"
 #include "posix-fs-canonicalise.hh"
 
@@ -175,8 +176,12 @@ static void opAdd(Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
 
+    PosixSourceAccessor accessor;
     for (auto & i : opArgs)
-        cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i)));
+        cout << fmt("%s\n", store->printStorePath(store->addToStore(
+            std::string(baseNameOf(i)),
+            accessor,
+            CanonPath::fromCwd(i))));
 }
 
 
@@ -196,8 +201,14 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
     HashAlgorithm hashAlgo = parseHashAlgo(opArgs.front());
     opArgs.pop_front();
 
+    PosixSourceAccessor accessor;
     for (auto & i : opArgs)
-        std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), i, method, hashAlgo).path));
+        std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(
+            baseNameOf(i),
+            accessor,
+            CanonPath::fromCwd(i),
+            method,
+            hashAlgo).path));
 }
 
 
@@ -541,7 +552,10 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise)
             if (canonicalise)
                 canonicalisePathMetaData(store->printStorePath(info->path), {});
             if (!hashGiven) {
-                HashResult hash = hashPath(HashAlgorithm::SHA256, store->printStorePath(info->path));
+                HashResult hash = hashPath(
+                    *store->getFSAccessor(false), CanonPath { store->printStorePath(info->path) },
+
+                    FileIngestionMethod::Recursive, HashAlgorithm::SHA256);
                 info->narHash = hash.first;
                 info->narSize = hash.second;
             }
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index 02de796b5..64a43ecfa 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -2,6 +2,7 @@
 #include "common-args.hh"
 #include "store-api.hh"
 #include "archive.hh"
+#include "posix-source-accessor.hh"
 
 using namespace nix;
 
@@ -20,7 +21,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
 {
     Path path;
     std::optional<std::string> namePart;
-    FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive;
+    ContentAddressMethod caMethod = FileIngestionMethod::Recursive;
 
     CmdAddToStore()
     {
@@ -48,7 +49,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
             )",
             .labels = {"hash-mode"},
             .handler = {[this](std::string s) {
-                this->ingestionMethod = parseIngestionMethod(s);
+                this->caMethod = parseIngestionMethod(s);
             }},
         });
     }
@@ -57,36 +58,17 @@ struct CmdAddToStore : MixDryRun, StoreCommand
     {
         if (!namePart) namePart = baseNameOf(path);
 
-        StringSink sink;
-        dumpPath(path, sink);
+        PosixSourceAccessor accessor;
 
-        auto narHash = hashString(HashAlgorithm::SHA256, sink.s);
+        auto path2 = CanonPath::fromCwd(path);
 
-        Hash hash = narHash;
-        if (ingestionMethod == FileIngestionMethod::Flat) {
-            HashSink hsink(HashAlgorithm::SHA256);
-            readFile(path, hsink);
-            hash = hsink.finish().first;
-        }
+        auto storePath = dryRun
+            ? store->computeStorePath(
+                *namePart, accessor, path2, caMethod, HashAlgorithm::SHA256, {}).first
+            : store->addToStoreSlow(
+                *namePart, accessor, path2, caMethod, HashAlgorithm::SHA256, {}).path;
 
-        ValidPathInfo info {
-            *store,
-            std::move(*namePart),
-            FixedOutputInfo {
-                .method = std::move(ingestionMethod),
-                .hash = std::move(hash),
-                .references = {},
-            },
-            narHash,
-        };
-        info.narSize = sink.s.size();
-
-        if (!dryRun) {
-            auto source = StringSource(sink.s);
-            store->addToStore(info, source);
-        }
-
-        logger->cout("%s", store->printStorePath(info.path));
+        logger->cout("%s", store->printStorePath(storePath));
     }
 };
 
@@ -110,7 +92,7 @@ struct CmdAddFile : CmdAddToStore
 {
     CmdAddFile()
     {
-        ingestionMethod = FileIngestionMethod::Flat;
+        caMethod = FileIngestionMethod::Flat;
     }
 
     std::string description() override
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 0bba3b7d2..83694306e 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -5,6 +5,7 @@
 #include "shared.hh"
 #include "references.hh"
 #include "archive.hh"
+#include "posix-source-accessor.hh"
 
 using namespace nix;
 
@@ -88,14 +89,8 @@ struct CmdHashBase : Command
             else
                 hashSink = std::make_unique<HashSink>(ha);
 
-            switch (mode) {
-            case FileIngestionMethod::Flat:
-                readFile(path, *hashSink);
-                break;
-            case FileIngestionMethod::Recursive:
-                dumpPath(path, *hashSink);
-                break;
-            }
+            PosixSourceAccessor accessor;
+            dumpPath(accessor, CanonPath::fromCwd(path), *hashSink, mode);
 
             Hash h = hashSink->finish().first;
             if (truncate && h.hashSize > 20) h = compressHash(h, 20);
diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc
index bbfeb8aa4..b5d619006 100644
--- a/src/nix/prefetch.cc
+++ b/src/nix/prefetch.cc
@@ -9,6 +9,7 @@
 #include "attr-path.hh"
 #include "eval-inline.hh"
 #include "legacy.hh"
+#include "posix-source-accessor.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -122,7 +123,11 @@ std::tuple<StorePath, Hash> prefetchFile(
         Activity act(*logger, lvlChatty, actUnknown,
             fmt("adding '%s' to the store", url));
 
-        auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashAlgo, expectedHash);
+        PosixSourceAccessor accessor;
+        auto info = store->addToStoreSlow(
+            *name,
+            accessor, CanonPath::fromCwd(tmpFile),
+            ingestionMethod, hashAlgo, {}, expectedHash);
         storePath = info.path;
         assert(info.ca);
         hash = info.ca->hash;
diff --git a/tests/unit/libexpr/primops.cc b/tests/unit/libexpr/primops.cc
index 31b1b49ae..6d7649b3c 100644
--- a/tests/unit/libexpr/primops.cc
+++ b/tests/unit/libexpr/primops.cc
@@ -604,7 +604,7 @@ namespace nix {
         ASSERT_THAT(v, IsStringEq("401b09eab3c013d4ca54922bb802bec8fd5318192b0a75f201d8b3727429080fb337591abd3e44453b954555b7a0812e1081c39b740293f765eae731f5a65ed1"));
     }
 
-    TEST_F(PrimOpTest, hashStringInvalidHashType) {
+    TEST_F(PrimOpTest, hashStringInvalidHashAlgorithm) {
         ASSERT_THROW(eval("builtins.hashString \"foobar\" \"asdf\""), Error);
     }
 

From ed26b186fbed9e5f8df2453a5b1aec0c18b11401 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 8 Nov 2023 21:11:48 -0500
Subject: [PATCH 251/421] Remove now-redundant text-hashing store methods

`addTextToStore` and `computeStorePathFromDump` are now redundant.

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 src/libexpr/primops.cc                      | 10 ++-
 src/libstore/binary-cache-store.cc          | 71 +++++++++++----------
 src/libstore/binary-cache-store.hh          |  6 --
 src/libstore/build/local-derivation-goal.cc | 11 ----
 src/libstore/content-address.hh             |  8 +--
 src/libstore/daemon.cc                      |  5 +-
 src/libstore/derivations.cc                 | 10 ++-
 src/libstore/dummy-store.cc                 |  7 --
 src/libstore/legacy-ssh-store.hh            |  7 --
 src/libstore/local-store.cc                 | 52 ---------------
 src/libstore/local-store.hh                 |  6 --
 src/libstore/remote-store.cc                | 10 ---
 src/libstore/remote-store.hh                |  6 --
 src/libstore/store-api.cc                   | 34 +++-------
 src/libstore/store-api.hh                   | 10 ---
 src/libstore/store-dir-config.hh            | 23 -------
 src/nix-env/user-env.cc                     | 13 ++--
 src/nix/develop.cc                          |  6 +-
 18 files changed, 83 insertions(+), 212 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 75ee1e38d..8b689f0c8 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -2072,8 +2072,14 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val
     }
 
     auto storePath = settings.readOnlyMode
-        ? state.store->computeStorePathForText(name, contents, refs)
-        : state.store->addTextToStore(name, contents, refs, state.repair);
+        ? state.store->makeFixedOutputPathFromCA(name, TextInfo {
+            .hash = hashString(HashAlgorithm::SHA256, contents),
+            .references = std::move(refs),
+        })
+        : ({
+            StringSource s { contents };
+            state.store->addToStoreFromDump(s, name, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, state.repair);
+        });
 
     /* Note: we don't need to add `context' to the context of the
        result, since `storePath' itself has references to the paths
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 19aa283fc..8a3052433 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -12,6 +12,7 @@
 #include "thread-pool.hh"
 #include "callback.hh"
 #include "signals.hh"
+#include "archive.hh"
 
 #include <chrono>
 #include <future>
@@ -308,15 +309,47 @@ StorePath BinaryCacheStore::addToStoreFromDump(
     const StorePathSet & references,
     RepairFlag repair)
 {
-    if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256)
-        unsupported("addToStoreFromDump");
-    return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
+    std::optional<Hash> caHash;
+    std::string nar;
+
+    if (auto * dump2p = dynamic_cast<StringSource *>(&dump)) {
+        auto & dump2 = *dump2p;
+        // Hack, this gives us a "replayable" source so we can compute
+        // multiple hashes more easily.
+        caHash = hashString(HashAlgorithm::SHA256, dump2.s);
+        switch (method.getFileIngestionMethod()) {
+        case FileIngestionMethod::Recursive:
+            // The dump is already NAR in this case, just use it.
+            nar = dump2.s;
+            break;
+        case FileIngestionMethod::Flat:
+            // The dump is Flat, so we need to convert it to NAR with a
+            // single file.
+            StringSink s;
+            dumpString(dump2.s, s);
+            nar = std::move(s.s);
+            break;
+        }
+    } else {
+        // Otherwise, we have to do th same hashing as NAR so our single
+        // hash will suffice for both purposes.
+        if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256)
+            unsupported("addToStoreFromDump");
+    }
+    StringSource narDump { nar };
+
+    // Use `narDump` if we wrote to `nar`.
+    Source & narDump2 = nar.size() > 0
+        ? static_cast<Source &>(narDump)
+        : dump;
+
+    return addToStoreCommon(narDump2, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info {
             *this,
             name,
             ContentAddressWithReferences::fromParts(
                 method,
-                nar.first,
+                caHash ? *caHash : nar.first,
                 {
                     .others = references,
                     // caller is not capable of creating a self-reference, because this is content-addressed without modulus
@@ -440,36 +473,6 @@ StorePath BinaryCacheStore::addToStore(
     })->path;
 }
 
-StorePath BinaryCacheStore::addTextToStore(
-    std::string_view name,
-    std::string_view s,
-    const StorePathSet & references,
-    RepairFlag repair)
-{
-    auto textHash = hashString(HashAlgorithm::SHA256, s);
-    auto path = makeTextPath(name, TextInfo { { textHash }, references });
-
-    if (!repair && isValidPath(path))
-        return path;
-
-    StringSink sink;
-    dumpString(s, sink);
-    StringSource source(sink.s);
-    return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
-        ValidPathInfo info {
-            *this,
-            std::string { name },
-            TextInfo {
-                .hash = textHash,
-                .references = references,
-            },
-            nar.first,
-        };
-        info.narSize = nar.second;
-        return info;
-    })->path;
-}
-
 void BinaryCacheStore::queryRealisationUncached(const DrvOutput & id,
     Callback<std::shared_ptr<const Realisation>> callback) noexcept
 {
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index dbe4ac180..98e43ee6a 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -141,12 +141,6 @@ public:
         PathFilter & filter,
         RepairFlag repair) override;
 
-    StorePath addTextToStore(
-        std::string_view name,
-        std::string_view s,
-        const StorePathSet & references,
-        RepairFlag repair) override;
-
     void registerDrvOutput(const Realisation & info) override;
 
     void queryRealisationUncached(const DrvOutput &,
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index e4828dd2f..b01d9e237 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -1308,17 +1308,6 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In
         goal.addDependency(info.path);
     }
 
-    StorePath addTextToStore(
-        std::string_view name,
-        std::string_view s,
-        const StorePathSet & references,
-        RepairFlag repair = NoRepair) override
-    {
-        auto path = next->addTextToStore(name, s, references, repair);
-        goal.addDependency(path);
-        return path;
-    }
-
     StorePath addToStoreFromDump(
         Source & dump,
         std::string_view name,
diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh
index 6863ad260..f0973412b 100644
--- a/src/libstore/content-address.hh
+++ b/src/libstore/content-address.hh
@@ -109,11 +109,11 @@ struct ContentAddressMethod
  * serialisation methods (flat file vs NAR). Thus, ‘ca’ has one of the
  * following forms:
  *
- * - ‘text:sha256:<sha256 hash of file contents>’: For paths
- *   computed by Store::makeTextPath() / Store::addTextToStore().
+ * - `TextIngestionMethod`:
+ *   ‘text:sha256:<sha256 hash of file contents>’
  *
- * - ‘fixed:<r?>:<ht>:<h>’: For paths computed by
- *   Store::makeFixedOutputPath() / Store::addToStore().
+ * - `FixedIngestionMethod`:
+ *   ‘fixed:<r?>:<hash type>:<hash of file contents>’
  */
 struct ContentAddress
 {
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 574263c68..923ea6447 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -483,7 +483,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         std::string s = readString(from);
         auto refs = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
         logger->startWork();
-        auto path = store->addTextToStore(suffix, s, refs, NoRepair);
+        auto path = ({
+            StringSource source { s };
+            store->addToStoreFromDump(source, suffix, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, NoRepair);
+        });
         logger->stopWork();
         to << store->printStorePath(path);
         break;
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index c35150b57..8a7d660ff 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -143,8 +143,14 @@ StorePath writeDerivation(Store & store,
     auto suffix = std::string(drv.name) + drvExtension;
     auto contents = drv.unparse(store, false);
     return readOnly || settings.readOnlyMode
-        ? store.computeStorePathForText(suffix, contents, references)
-        : store.addTextToStore(suffix, contents, references, repair);
+        ? store.makeFixedOutputPathFromCA(suffix, TextInfo {
+            .hash = hashString(HashAlgorithm::SHA256, contents),
+            .references = std::move(references),
+        })
+        : ({
+            StringSource s { contents };
+            store.addToStoreFromDump(s, suffix, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair);
+        });
 }
 
 
diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc
index 821cda399..f52a309d1 100644
--- a/src/libstore/dummy-store.cc
+++ b/src/libstore/dummy-store.cc
@@ -58,13 +58,6 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
         RepairFlag repair, CheckSigsFlag checkSigs) override
     { unsupported("addToStore"); }
 
-    StorePath addTextToStore(
-        std::string_view name,
-        std::string_view s,
-        const StorePathSet & references,
-        RepairFlag repair) override
-    { unsupported("addTextToStore"); }
-
     void narFromPath(const StorePath & path, Sink & sink) override
     { unsupported("narFromPath"); }
 
diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh
index 8b142ba2a..c5a3ce677 100644
--- a/src/libstore/legacy-ssh-store.hh
+++ b/src/libstore/legacy-ssh-store.hh
@@ -69,13 +69,6 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
         RepairFlag repair) override
     { unsupported("addToStore"); }
 
-    StorePath addTextToStore(
-        std::string_view name,
-        std::string_view s,
-        const StorePathSet & references,
-        RepairFlag repair) override
-    { unsupported("addTextToStore"); }
-
 private:
 
     void putBuildSettings(Connection & conn);
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index cd8bf24f8..df1de7752 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1257,58 +1257,6 @@ StorePath LocalStore::addToStoreFromDump(
 }
 
 
-StorePath LocalStore::addTextToStore(
-    std::string_view name,
-    std::string_view s,
-    const StorePathSet & references, RepairFlag repair)
-{
-    auto hash = hashString(HashAlgorithm::SHA256, s);
-    auto dstPath = makeTextPath(name, TextInfo {
-        .hash = hash,
-        .references = references,
-    });
-
-    addTempRoot(dstPath);
-
-    if (repair || !isValidPath(dstPath)) {
-
-        auto realPath = Store::toRealPath(dstPath);
-
-        PathLocks outputLock({realPath});
-
-        if (repair || !isValidPath(dstPath)) {
-
-            deletePath(realPath);
-
-            autoGC();
-
-            writeFile(realPath, s);
-
-            canonicalisePathMetaData(realPath, {});
-
-            StringSink sink;
-            dumpString(s, sink);
-            auto narHash = hashString(HashAlgorithm::SHA256, sink.s);
-
-            optimisePath(realPath, repair);
-
-            ValidPathInfo info { dstPath, narHash };
-            info.narSize = sink.s.size();
-            info.references = references;
-            info.ca = {
-                .method = TextIngestionMethod {},
-                .hash = hash,
-            };
-            registerValidPath(info);
-        }
-
-        outputLock.setDeletion(true);
-    }
-
-    return dstPath;
-}
-
-
 /* Create a temporary directory in the store that won't be
    garbage-collected until the returned FD is closed. */
 std::pair<Path, AutoCloseFD> LocalStore::createTempDirInStore()
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index a8323fe5a..ba56d3ead 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -185,12 +185,6 @@ public:
         const StorePathSet & references,
         RepairFlag repair) override;
 
-    StorePath addTextToStore(
-        std::string_view name,
-        std::string_view s,
-        const StorePathSet & references,
-        RepairFlag repair) override;
-
     void addTempRoot(const StorePath & path) override;
 
 private:
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 567776b67..4d0113594 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -608,16 +608,6 @@ void RemoteStore::addMultipleToStore(
 }
 
 
-StorePath RemoteStore::addTextToStore(
-    std::string_view name,
-    std::string_view s,
-    const StorePathSet & references,
-    RepairFlag repair)
-{
-    StringSource source(s);
-    return addCAToStore(source, name, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair)->path;
-}
-
 void RemoteStore::registerDrvOutput(const Realisation & info)
 {
     auto conn(getConnection());
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 68824a737..87704985b 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -106,12 +106,6 @@ public:
         RepairFlag repair,
         CheckSigsFlag checkSigs) override;
 
-    StorePath addTextToStore(
-        std::string_view name,
-        std::string_view s,
-        const StorePathSet & references,
-        RepairFlag repair) override;
-
     void registerDrvOutput(const Realisation & info) override;
 
     void queryRealisationUncached(const DrvOutput &,
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 5b4c6c765..c2516afb5 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -205,25 +205,19 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed
 }
 
 
-StorePath StoreDirConfig::makeTextPath(std::string_view name, const TextInfo & info) const
-{
-    assert(info.hash.algo == HashAlgorithm::SHA256);
-    return makeStorePath(
-        makeType(*this, "text", StoreReferences {
-            .others = info.references,
-            .self = false,
-        }),
-        info.hash,
-        name);
-}
-
-
 StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const
 {
     // New template
     return std::visit(overloaded {
         [&](const TextInfo & ti) {
-            return makeTextPath(name, ti);
+            assert(ti.hash.algo == HashAlgorithm::SHA256);
+            return makeStorePath(
+                makeType(*this, "text", StoreReferences {
+                    .others = ti.references,
+                    .self = false,
+                }),
+                ti.hash,
+                name);
         },
         [&](const FixedOutputInfo & foi) {
             return makeFixedOutputPath(name, foi);
@@ -257,18 +251,6 @@ std::pair<StorePath, Hash> StoreDirConfig::computeStorePath(
 }
 
 
-StorePath StoreDirConfig::computeStorePathForText(
-    std::string_view name,
-    std::string_view s,
-    const StorePathSet & references) const
-{
-    return makeTextPath(name, TextInfo {
-        .hash = hashString(HashAlgorithm::SHA256, s),
-        .references = references,
-    });
-}
-
-
 StorePath Store::addToStore(
     std::string_view name,
     SourceAccessor & accessor,
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index fc0a82a73..96a7ebd7b 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -468,16 +468,6 @@ public:
         RepairFlag repair = NoRepair)
     { unsupported("addToStoreFromDump"); }
 
-    /**
-     * Like addToStore, but the contents written to the output path is a
-     * regular file containing the given string.
-     */
-    virtual StorePath addTextToStore(
-        std::string_view name,
-        std::string_view s,
-        const StorePathSet & references,
-        RepairFlag repair = NoRepair) = 0;
-
     /**
      * Add a mapping indicating that `deriver!outputName` maps to the output path
      * `output`.
diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh
index 0fc8ded9c..7ca8c2665 100644
--- a/src/libstore/store-dir-config.hh
+++ b/src/libstore/store-dir-config.hh
@@ -86,8 +86,6 @@ struct StoreDirConfig : public Config
 
     StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const;
 
-    StorePath makeTextPath(std::string_view name, const TextInfo & info) const;
-
     StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const;
 
     /**
@@ -102,27 +100,6 @@ struct StoreDirConfig : public Config
         HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
         const StorePathSet & references = {},
         PathFilter & filter = defaultPathFilter) const;
-
-    /**
-     * Preparatory part of addTextToStore().
-     *
-     * !!! Computation of the path should take the references given to
-     * addTextToStore() into account, otherwise we have a (relatively
-     * minor) security hole: a caller can register a source file with
-     * bogus references.  If there are too many references, the path may
-     * not be garbage collected when it has to be (not really a problem,
-     * the caller could create a root anyway), or it may be garbage
-     * collected when it shouldn't be (more serious).
-     *
-     * Hashing the references would solve this (bogus references would
-     * simply yield a different store path, so other users wouldn't be
-     * affected), but it has some backwards compatibility issues (the
-     * hashing scheme changes), so I'm not doing that for now.
-     */
-    StorePath computeStorePathForText(
-        std::string_view name,
-        std::string_view s,
-        const StorePathSet & references) const;
 };
 
 }
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index 34f6bd005..5d01fbf10 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -104,10 +104,15 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
     /* Also write a copy of the list of user environment elements to
        the store; we need it for future modifications of the
        environment. */
-    std::ostringstream str;
-    manifest.print(state.symbols, str, true);
-    auto manifestFile = state.store->addTextToStore("env-manifest.nix",
-        str.str(), references);
+    auto manifestFile = ({
+        std::ostringstream str;
+        manifest.print(state.symbols, str, true);
+        // TODO with C++20 we can use str.view() instead and avoid copy.
+        std::string str2 = str.str();
+        StringSource source { str2 };
+        state.store->addToStoreFromDump(
+            source, "env-manifest.nix", TextIngestionMethod {}, HashAlgorithm::SHA256, references);
+    });
 
     /* Get the environment builder expression. */
     Value envBuilder;
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 606b044b0..8db2de491 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -223,7 +223,11 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore
     if (builder != "bash")
         throw Error("'nix develop' only works on derivations that use 'bash' as their builder");
 
-    auto getEnvShPath = evalStore->addTextToStore("get-env.sh", getEnvSh, {});
+    auto getEnvShPath = ({
+        StringSource source { getEnvSh };
+        evalStore->addToStoreFromDump(
+            source, "get-env.sh", TextIngestionMethod {}, HashAlgorithm::SHA256, {});
+    });
 
     drv.args = {store->printStorePath(getEnvShPath)};
 

From b1c559eabccc8890c74f4e520b89e800f6e7ef7e Mon Sep 17 00:00:00 2001
From: tomberek <tomberek@users.noreply.github.com>
Date: Mon, 18 Dec 2023 10:45:57 -0500
Subject: [PATCH 252/421] docs: add link to project board to PRs (#9630)

* docs: add link to project board to PRs

* Update .github/PULL_REQUEST_TEMPLATE.md

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>

* fix wording

* add note on the process

---------

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 .github/PULL_REQUEST_TEMPLATE.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 217b19108..d12a4d36c 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -10,6 +10,8 @@
 
 <!-- Large change: Provide instructions to reviewers how to read the diff. -->
 
-# Priorities
+# Priorities and Process
 
 Add :+1: to [pull requests you find important](https://github.com/NixOS/nix/pulls?q=is%3Aopen+sort%3Areactions-%2B1-desc).
+
+The Nix maintainer team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19) to [schedule and track reviews](https://github.com/NixOS/nix/tree/master/maintainers#project-board-protocol). 

From 7feabf7d44c960563350a246358d4e36bd598d60 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 14 Dec 2023 00:05:03 -0500
Subject: [PATCH 253/421] Split `--disable-tests`, fix cross builds

It might seem obnoxious to have yet more configure flags, but I found
controlling both the unit and functional tests with one flag was quite
confusing because they are so different:

- unit tests depending on building, functional tests don't (e.g. when
  we test already-built Nix)

- unit tests can be installed, functional tests cannot

- unit tests neeed extra libraries (GTest, RapidCheck), functional
  tests need extra executables (jq).

- unit tests are run by `make check`, functional tests are run by `make
  installcheck`

Really on a technical level, they seem wholly independent. Only on a
human level ("they are both are tests") do they have anything in common.

I had messed up the logic in cross builds because of this. Now I
split the flag in two (and cleaned up a few other inconsistencies), and
the logic fixed itself.

Co-Authored-By: Robert Hensing <roberth@users.noreply.github.com>
---
 Makefile                                      | 30 ++++++--
 Makefile.config.in                            |  7 +-
 configure.ac                                  | 43 ++++++++----
 doc/internal-api/local.mk                     | 14 +---
 doc/manual/local.mk                           |  6 +-
 doc/manual/src/contributing/hacking.md        |  5 +-
 .../src/installation/prerequisites-source.md  |  2 +-
 mk/disable-tests.mk                           | 12 ----
 package.nix                                   | 69 ++++++++-----------
 9 files changed, 93 insertions(+), 95 deletions(-)
 delete mode 100644 mk/disable-tests.mk

diff --git a/Makefile b/Makefile
index 3dae8b394..c62216df8 100644
--- a/Makefile
+++ b/Makefile
@@ -24,7 +24,7 @@ makefiles = \
   misc/upstart/local.mk
 endif
 
-ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes)
+ifeq ($(ENABLE_UNIT_TESTS), yes)
 makefiles += \
   tests/unit/libutil/local.mk \
   tests/unit/libutil-support/local.mk \
@@ -32,9 +32,14 @@ makefiles += \
   tests/unit/libstore-support/local.mk \
   tests/unit/libexpr/local.mk \
   tests/unit/libexpr-support/local.mk
+else
+.PHONY: check
+check:
+	@echo "Unit tests are disabled. Configure without '--disable-unit-tests', or avoid calling 'make check'."
+	@exit 1
 endif
 
-ifeq ($(ENABLE_TESTS), yes)
+ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes)
 makefiles += \
   tests/functional/local.mk \
   tests/functional/ca/local.mk \
@@ -42,8 +47,10 @@ makefiles += \
   tests/functional/test-libstoreconsumer/local.mk \
   tests/functional/plugins/local.mk
 else
-makefiles += \
-  mk/disable-tests.mk
+.PHONY: installcheck
+installcheck:
+	@echo "Functional tests are disabled. Configure without '--disable-functional-tests', or avoid calling 'make installcheck'."
+	@exit 1
 endif
 
 OPTIMIZE = 1
@@ -59,9 +66,22 @@ include mk/lib.mk
 
 # Must be included after `mk/lib.mk` so rules refer to variables defined
 # by the library. Rules are not "lazy" like variables, unfortunately.
-ifeq ($(ENABLE_BUILD), yes)
+ifeq ($(ENABLE_DOC_GEN),yes)
 $(eval $(call include-sub-makefile, doc/manual/local.mk))
+else
+.PHONY: manual-html manpages
+manual-html manpages:
+	@echo "Generated docs are disabled. Configure without '--disable-doc-gen', or avoid calling 'make manpages' and 'make manual-html'."
+	@exit 1
 endif
+
+ifeq ($(ENABLE_INTERNAL_API_DOCS),yes)
 $(eval $(call include-sub-makefile, doc/internal-api/local.mk))
+else
+.PHONY: internal-api-html
+internal-api-html:
+	@echo "Internal API docs are disabled. Configure with '--enable-internal-api-docs', or avoid calling 'make internal-api-html'."
+	@exit 1
+endif
 
 GLOBAL_CXXFLAGS += -g -Wall -include $(buildprefix)config.h -std=c++2a -I src
diff --git a/Makefile.config.in b/Makefile.config.in
index c85e028c2..21a9f41ec 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -9,8 +9,11 @@ CXXFLAGS = @CXXFLAGS@
 CXXLTO = @CXXLTO@
 EDITLINE_LIBS = @EDITLINE_LIBS@
 ENABLE_BUILD = @ENABLE_BUILD@
+ENABLE_DOC_GEN = @ENABLE_DOC_GEN@
+ENABLE_FUNCTIONAL_TESTS = @ENABLE_FUNCTIONAL_TESTS@
+ENABLE_INTERNAL_API_DOCS = @ENABLE_INTERNAL_API_DOCS@
 ENABLE_S3 = @ENABLE_S3@
-ENABLE_TESTS = @ENABLE_TESTS@
+ENABLE_UNIT_TESTS = @ENABLE_UNIT_TESTS@
 GTEST_LIBS = @GTEST_LIBS@
 HAVE_LIBCPUID = @HAVE_LIBCPUID@
 HAVE_SECCOMP = @HAVE_SECCOMP@
@@ -36,12 +39,10 @@ checkbindir = @checkbindir@
 checklibdir = @checklibdir@
 datadir = @datadir@
 datarootdir = @datarootdir@
-doc_generate = @doc_generate@
 docdir = @docdir@
 embedded_sandbox_shell = @embedded_sandbox_shell@
 exec_prefix = @exec_prefix@
 includedir = @includedir@
-internal_api_docs = @internal_api_docs@
 libdir = @libdir@
 libexecdir = @libexecdir@
 localstatedir = @localstatedir@
diff --git a/configure.ac b/configure.ac
index a949f9df2..1bc4f17b0 100644
--- a/configure.ac
+++ b/configure.ac
@@ -138,20 +138,38 @@ AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]),
   ENABLE_BUILD=$enableval, ENABLE_BUILD=yes)
 AC_SUBST(ENABLE_BUILD)
 
-# Building without tests is useful for bootstrapping with a smaller footprint
+# Building without unit tests is useful for bootstrapping with a smaller footprint
 # or running the tests in a separate derivation. Otherwise, we do compile and
 # run them.
-AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]),
-  ENABLE_TESTS=$enableval, ENABLE_TESTS=yes)
-AC_SUBST(ENABLE_TESTS)
 
-# Building without API docs is the default as Nix' C++ interfaces are internal and unstable.
-AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]),
-  internal_api_docs=$enableval, internal_api_docs=no)
-AC_SUBST(internal_api_docs)
+AC_ARG_ENABLE(unit-tests, AS_HELP_STRING([--disable-unit-tests],[Do not build the tests]),
+  ENABLE_UNIT_TESTS=$enableval, ENABLE_UNIT_TESTS=$ENABLE_BUILD)
+AC_SUBST(ENABLE_UNIT_TESTS)
 
 AS_IF(
-  [test "$ENABLE_BUILD" == "yes" || test "$ENABLE_TEST" == "yes"],
+  [test "$ENABLE_BUILD" == "no" && test "$ENABLE_UNIT_TESTS" == "yes"],
+  [AC_MSG_ERROR([Cannot enable unit tests when building overall is disabled. Please do not pass '--enable-unit-tests' or do not pass '--disable-build'.])])
+
+AC_ARG_ENABLE(functional-tests, AS_HELP_STRING([--disable-functional-tests],[Do not build the tests]),
+  ENABLE_FUNCTIONAL_TESTS=$enableval, ENABLE_FUNCTIONAL_TESTS=yes)
+AC_SUBST(ENABLE_FUNCTIONAL_TESTS)
+
+# documentation generation switch
+AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation generation]),
+  ENABLE_DOC_GEN=$enableval, ENABLE_DOC_GEN=$ENABLE_BUILD)
+AC_SUBST(ENABLE_DOC_GEN)
+
+AS_IF(
+  [test "$ENABLE_BUILD" == "no" && test "$ENABLE_GENERATED_DOCS" == "yes"],
+  [AC_MSG_ERROR([Cannot enable generated docs when building overall is disabled. Please do not pass '--enable-doc-gen' or do not pass '--disable-build'.])])
+
+# Building without API docs is the default as Nix' C++ interfaces are internal and unstable.
+AC_ARG_ENABLE(internal-api-docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]),
+  ENABLE_INTERNAL_API_DOCS=$enableval, ENABLE_INTERNAL_API_DOCS=no)
+AC_SUBST(ENABLE_INTERNAL_API_DOCS)
+
+AS_IF(
+  [test "$ENABLE_FUNCTIONAL_TESTS" == "yes" || test "$ENABLE_DOC_GEN" == "yes"],
   [NEED_PROG(jq, jq)])
 
 AS_IF([test "$ENABLE_BUILD" == "yes"],[
@@ -317,7 +335,7 @@ if test "$gc" = yes; then
   AC_DEFINE(HAVE_BOEHMGC, 1, [Whether to use the Boehm garbage collector.])
 fi
 
-AS_IF([test "$ENABLE_TESTS" == "yes"],[
+AS_IF([test "$ENABLE_UNIT_TESTS" == "yes"],[
 
 # Look for gtest.
 PKG_CHECK_MODULES([GTEST], [gtest_main])
@@ -349,11 +367,6 @@ AC_LANG_POP(C++)
 # Look for nlohmann/json.
 PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9])
 
-# documentation generation switch
-AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation generation]),
-  doc_generate=$enableval, doc_generate=yes)
-AC_SUBST(doc_generate)
-
 
 # Look for lowdown library.
 PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"])
diff --git a/doc/internal-api/local.mk b/doc/internal-api/local.mk
index 890f341b7..bf2c4dede 100644
--- a/doc/internal-api/local.mk
+++ b/doc/internal-api/local.mk
@@ -1,19 +1,7 @@
-.PHONY: internal-api-html
-
-ifeq ($(internal_api_docs), yes)
-
 $(docdir)/internal-api/html/index.html $(docdir)/internal-api/latex: $(d)/doxygen.cfg
 	mkdir -p $(docdir)/internal-api
 	{ cat $< ; echo "OUTPUT_DIRECTORY=$(docdir)/internal-api" ; } | doxygen -
 
 # Generate the HTML API docs for Nix's unstable internal interfaces.
+.PHONY: internal-api-html
 internal-api-html: $(docdir)/internal-api/html/index.html
-
-else
-
-# Make a nicer error message
-internal-api-html:
-	@echo "Internal API docs are disabled. Configure with '--enable-internal-api-docs', or avoid calling 'make internal-api-html'."
-	@exit 1
-
-endif
diff --git a/doc/manual/local.mk b/doc/manual/local.mk
index 456000d3d..b77168885 100644
--- a/doc/manual/local.mk
+++ b/doc/manual/local.mk
@@ -1,5 +1,3 @@
-ifeq ($(doc_generate),yes)
-
 # The version of Nix used to generate the doc. Can also be
 # `$(nix_INSTALL_PATH)` or just `nix` (to grap ambient from the `PATH`),
 # if one prefers.
@@ -180,6 +178,8 @@ manual-html: $(docdir)/manual/index.html
 install: $(docdir)/manual/index.html
 
 # Generate 'nix' manpages.
+.PHONY: manpages
+manpages: $(mandir)/man1/nix3-manpages
 install: $(mandir)/man1/nix3-manpages
 man: doc/manual/generated/man1/nix3-manpages
 all: doc/manual/generated/man1/nix3-manpages
@@ -225,5 +225,3 @@ $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/
 	@rm -rf $(DESTDIR)$(docdir)/manual
 	@mv $(DESTDIR)$(docdir)/manual.tmp/html $(DESTDIR)$(docdir)/manual
 	@rm -rf $(DESTDIR)$(docdir)/manual.tmp
-
-endif
diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index 421ac981c..9478c424d 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -67,9 +67,10 @@ You can also build Nix for one of the [supported platforms](#platforms).
 ## Makefile variables
 
 - `ENABLE_BUILD=yes` to enable building the C++ code.
-- `ENABLE_TESTS=yes` to enable building the tests.
+- `ENABLE_DOC_GEN=yes` to enable building the documentation (manual, man pages, etc.).
+- `ENABLE_FUNCTIONAL_TESTS=yes` to enable building the functional tests.
+- `ENABLE_UNIT_TESTS=yes` to enable building the unit tests.
 - `OPTIMIZE=1` to enable optimizations.
-- `doc_generate=yes` to enable building the documentation (manual, man pages, etc.).
 
   The docs can take a while to build, so you may want to disable this for local development.
 
diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md
index d4babf1ea..807e82517 100644
--- a/doc/manual/src/installation/prerequisites-source.md
+++ b/doc/manual/src/installation/prerequisites-source.md
@@ -72,7 +72,7 @@
     This is an optional dependency and can be disabled
     by providing a `--disable-cpuid` to the `configure` script.
 
-  - Unless `./configure --disable-tests` is specified, GoogleTest (GTest) and
+  - Unless `./configure --disable-unit-tests` is specified, GoogleTest (GTest) and
     RapidCheck are required, which are available at
     <https://google.github.io/googletest/> and
     <https://github.com/emil-e/rapidcheck> respectively.
diff --git a/mk/disable-tests.mk b/mk/disable-tests.mk
deleted file mode 100644
index f72f84412..000000000
--- a/mk/disable-tests.mk
+++ /dev/null
@@ -1,12 +0,0 @@
-# This file is only active for `./configure --disable-tests`.
-# Running `make check` or `make installcheck` would indicate a mistake in the
-# caller.
-
-installcheck:
-	@echo "Tests are disabled. Configure without '--disable-tests', or avoid calling 'make installcheck'."
-	@exit 1
-
-# This currently has little effect.
-check:
-	@echo "Tests are disabled. Configure without '--disable-tests', or avoid calling 'make check'."
-	@exit 1
diff --git a/package.nix b/package.nix
index 24395b484..370820c40 100644
--- a/package.nix
+++ b/package.nix
@@ -104,30 +104,6 @@ let
     inherit doBuild doCheck doInstallCheck;
   };
 
-  filesets = {
-    baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.;
-
-    configureFiles = fileset.unions [
-      ./.version
-      ./configure.ac
-      ./m4
-      # TODO: do we really need README.md? It doesn't seem used in the build.
-      ./README.md
-    ];
-
-    topLevelBuildFiles = fileset.unions [
-      ./local.mk
-      ./Makefile
-      ./Makefile.config.in
-      ./mk
-    ];
-
-    functionalTestFiles = fileset.unions [
-      ./tests/functional
-      (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
-    ];
-  };
-
   mkDerivation =
     if withCoverageChecks
     then
@@ -151,32 +127,44 @@ mkDerivation (finalAttrs: let
   # to be run later, requiresthe unit tests to be built.
   buildUnitTests = doCheck || installUnitTests;
 
-  anySortOfTesting = buildUnitTests || doInstallCheck;
-
 in {
   inherit pname version;
 
   src =
     let
-
+      baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.;
     in
       fileset.toSource {
         root = ./.;
-        fileset = fileset.intersect filesets.baseFiles (fileset.unions ([
-          filesets.configureFiles
-          filesets.topLevelBuildFiles
-          ./doc/internal-api
+        fileset = fileset.intersect baseFiles (fileset.unions ([
+          # For configure
+          ./.version
+          ./configure.ac
+          ./m4
+          # TODO: do we really need README.md? It doesn't seem used in the build.
+          ./README.md
+          # For make, regardless of what we are building
+          ./local.mk
+          ./Makefile
+          ./Makefile.config.in
+          ./mk
+          (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
         ] ++ lib.optionals doBuild [
           ./boehmgc-coroutine-sp-fallback.diff
           ./doc
           ./misc
           ./precompiled-headers.h
           ./src
-          ./tests/unit
           ./COPYING
           ./scripts/local.mk
-        ] ++ lib.optionals anySortOfTesting [
-          filesets.functionalTestFiles
+        ] ++ lib.optionals buildUnitTests [
+          ./doc/manual
+        ] ++ lib.optionals enableInternalAPIDocs [
+          ./doc/internal-api
+        ] ++ lib.optionals buildUnitTests [
+          ./tests/unit
+        ] ++ lib.optionals doInstallCheck [
+          ./tests/functional
         ]));
       };
 
@@ -277,7 +265,8 @@ in {
   configureFlags = [
     "--sysconfdir=/etc"
     (lib.enableFeature doBuild "build")
-    (lib.enableFeature anySortOfTesting "tests")
+    (lib.enableFeature buildUnitTests "unit-tests")
+    (lib.enableFeature doInstallCheck "functional-tests")
     (lib.enableFeature enableInternalAPIDocs "internal-api-docs")
     (lib.enableFeature enableManual "doc-gen")
     (lib.enableFeature installUnitTests "install-unit-tests")
@@ -310,10 +299,7 @@ in {
   '';
 
   postInstall = lib.optionalString doBuild (
-    ''
-      mkdir -p $doc/nix-support
-      echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
-    '' + lib.optionalString stdenv.hostPlatform.isStatic ''
+    lib.optionalString stdenv.hostPlatform.isStatic ''
       mkdir -p $out/nix-support
       echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
     '' + lib.optionalString stdenv.isDarwin ''
@@ -322,7 +308,10 @@ in {
       $out/lib/libboost_context.dylib \
       $out/lib/libnixutil.dylib
     ''
-  ) + lib.optionalString enableInternalAPIDocs ''
+  ) + lib.optionalString enableManual ''
+    mkdir -p ''${!outputDoc}/nix-support
+    echo "doc manual ''${!outputDoc}/share/doc/nix/manual" >> ''${!outputDoc}/nix-support/hydra-build-products
+  '' + lib.optionalString enableInternalAPIDocs ''
     mkdir -p ''${!outputDoc}/nix-support
     echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products
   '';

From 123ef6a9967d5ca8ed4052d84128ff0e98950532 Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.e.helbling@outlook.com>
Date: Mon, 18 Dec 2023 10:19:25 -0800
Subject: [PATCH 254/421] Fix warnings when running checks

`nix flake check` had these warnings:

    trace: warning: Module argument `nodes.client.config` is deprecated. Use `nodes.client` instead.
    trace: warning: Module argument `nodes.client.config` is deprecated. Use `nodes.client` instead.
    trace: warning: The option `services.openssh.permitRootLogin' defined in `/nix/store/3m3hfpmbjdf4w39qfjami7ljhvhczay1-source/tests/nixos/nix-copy.nix' has been renamed to `services.openssh.settings.PermitRootLogin'.
    trace: warning: Module argument `nodes.http_dns.config` is deprecated. Use `nodes.http_dns` instead.
    trace: warning: Module argument `nodes.github.config` is deprecated. Use `nodes.github` instead.
    trace: warning: Module argument `nodes.sourcehut.config` is deprecated. Use `nodes.sourcehut` instead.
---
 tests/nixos/github-flakes.nix        | 2 +-
 tests/nixos/nix-copy.nix             | 2 +-
 tests/nixos/nss-preload.nix          | 4 ++--
 tests/nixos/remote-builds-ssh-ng.nix | 2 +-
 tests/nixos/remote-builds.nix        | 8 ++++----
 tests/nixos/sourcehut-flakes.nix     | 2 +-
 6 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix
index 62ae8871b..a51689445 100644
--- a/tests/nixos/github-flakes.nix
+++ b/tests/nixos/github-flakes.nix
@@ -144,7 +144,7 @@ in
           virtualisation.memorySize = 4096;
           nix.settings.substituters = lib.mkForce [ ];
           nix.extraOptions = "experimental-features = nix-command flakes";
-          networking.hosts.${(builtins.head nodes.github.config.networking.interfaces.eth1.ipv4.addresses).address} =
+          networking.hosts.${(builtins.head nodes.github.networking.interfaces.eth1.ipv4.addresses).address} =
             [ "channels.nixos.org" "api.github.com" "github.com" ];
           security.pki.certificateFiles = [ "${cert}/ca.crt" ];
         };
diff --git a/tests/nixos/nix-copy.nix b/tests/nixos/nix-copy.nix
index 2981cc2b8..7db5197aa 100644
--- a/tests/nixos/nix-copy.nix
+++ b/tests/nixos/nix-copy.nix
@@ -36,7 +36,7 @@ in {
       server =
         { config, pkgs, ... }:
         { services.openssh.enable = true;
-          services.openssh.permitRootLogin = "yes";
+          services.openssh.settings.PermitRootLogin = "yes";
 					users.users.root.password = "foobar";
           virtualisation.writableStore = true;
           virtualisation.additionalPaths = [ pkgB pkgC ];
diff --git a/tests/nixos/nss-preload.nix b/tests/nixos/nss-preload.nix
index cef62e95b..00505d114 100644
--- a/tests/nixos/nss-preload.nix
+++ b/tests/nixos/nss-preload.nix
@@ -84,8 +84,8 @@ in
     client = { lib, nodes, pkgs, ... }: {
       networking.useDHCP = false;
       networking.nameservers = [
-        (lib.head nodes.http_dns.config.networking.interfaces.eth1.ipv6.addresses).address
-        (lib.head nodes.http_dns.config.networking.interfaces.eth1.ipv4.addresses).address
+        (lib.head nodes.http_dns.networking.interfaces.eth1.ipv6.addresses).address
+        (lib.head nodes.http_dns.networking.interfaces.eth1.ipv4.addresses).address
       ];
       networking.interfaces.eth1.ipv6.addresses = [
         { address = "fd21::10"; prefixLength = 64; }
diff --git a/tests/nixos/remote-builds-ssh-ng.nix b/tests/nixos/remote-builds-ssh-ng.nix
index b59dde9bf..20a43803d 100644
--- a/tests/nixos/remote-builds-ssh-ng.nix
+++ b/tests/nixos/remote-builds-ssh-ng.nix
@@ -81,7 +81,7 @@ in
     client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'")
 
     # Perform a build
-    out = client.succeed("nix-build ${expr nodes.client.config 1} 2> build-output")
+    out = client.succeed("nix-build ${expr nodes.client 1} 2> build-output")
 
     # Verify that the build was done on the builder
     builder.succeed(f"test -e {out.strip()}")
diff --git a/tests/nixos/remote-builds.nix b/tests/nixos/remote-builds.nix
index 1c96cc787..ad7f509db 100644
--- a/tests/nixos/remote-builds.nix
+++ b/tests/nixos/remote-builds.nix
@@ -90,22 +90,22 @@ in
 
     # Perform a build and check that it was performed on the builder.
     out = client.succeed(
-      "nix-build ${expr nodes.client.config 1} 2> build-output",
+      "nix-build ${expr nodes.client 1} 2> build-output",
       "grep -q Hello build-output"
     )
     builder1.succeed(f"test -e {out}")
 
     # And a parallel build.
-    paths = client.succeed(r'nix-store -r $(nix-instantiate ${expr nodes.client.config 2})\!out $(nix-instantiate ${expr nodes.client.config 3})\!out')
+    paths = client.succeed(r'nix-store -r $(nix-instantiate ${expr nodes.client 2})\!out $(nix-instantiate ${expr nodes.client 3})\!out')
     out1, out2 = paths.split()
     builder1.succeed(f"test -e {out1} -o -e {out2}")
     builder2.succeed(f"test -e {out1} -o -e {out2}")
 
     # And a failing build.
-    client.fail("nix-build ${expr nodes.client.config 5}")
+    client.fail("nix-build ${expr nodes.client 5}")
 
     # Test whether the build hook automatically skips unavailable builders.
     builder1.block()
-    client.succeed("nix-build ${expr nodes.client.config 4}")
+    client.succeed("nix-build ${expr nodes.client 4}")
   '';
 }
diff --git a/tests/nixos/sourcehut-flakes.nix b/tests/nixos/sourcehut-flakes.nix
index 6e8d884a0..04f3590e1 100644
--- a/tests/nixos/sourcehut-flakes.nix
+++ b/tests/nixos/sourcehut-flakes.nix
@@ -108,7 +108,7 @@ in
               flake-registry = https://git.sr.ht/~NixOS/flake-registry/blob/master/flake-registry.json
             '';
             environment.systemPackages = [ pkgs.jq ];
-            networking.hosts.${(builtins.head nodes.sourcehut.config.networking.interfaces.eth1.ipv4.addresses).address} =
+            networking.hosts.${(builtins.head nodes.sourcehut.networking.interfaces.eth1.ipv4.addresses).address} =
               [ "git.sr.ht" ];
             security.pki.certificateFiles = [ "${cert}/ca.crt" ];
           };

From 1f7b62f123fde15b89746b6b1f73c40a8e927499 Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.e.helbling@outlook.com>
Date: Mon, 18 Dec 2023 10:36:18 -0800
Subject: [PATCH 255/421] Use `nix daemon` in the test suite

As part of the CLI stabilization effort, the last remaining checkbox (at
the moment) for `nix daemon` is that it "needs testing". This implements
the proposal of using `nix daemon` in place of `nix-daemon` in the test
suite.
---
 tests/functional/build-remote-trustless-should-pass-1.sh | 2 +-
 tests/functional/common/vars-and-functions.sh.in         | 4 ++--
 tests/functional/nix-daemon-untrusting.sh                | 2 +-
 tests/functional/store-info.sh                           | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/functional/build-remote-trustless-should-pass-1.sh b/tests/functional/build-remote-trustless-should-pass-1.sh
index 516bdf092..736e280e4 100644
--- a/tests/functional/build-remote-trustless-should-pass-1.sh
+++ b/tests/functional/build-remote-trustless-should-pass-1.sh
@@ -2,7 +2,7 @@ source common.sh
 
 # Remote trusts us
 file=build-hook.nix
-prog=nix-daemon
+prog='nix%20daemon'
 proto=ssh-ng
 
 source build-remote-trustless.sh
diff --git a/tests/functional/common/vars-and-functions.sh.in b/tests/functional/common/vars-and-functions.sh.in
index 848988af9..c25366481 100644
--- a/tests/functional/common/vars-and-functions.sh.in
+++ b/tests/functional/common/vars-and-functions.sh.in
@@ -95,7 +95,7 @@ startDaemon() {
     fi
     # Start the daemon, wait for the socket to appear.
     rm -f $NIX_DAEMON_SOCKET_PATH
-    PATH=$DAEMON_PATH nix-daemon &
+    PATH=$DAEMON_PATH nix --extra-experimental-features 'nix-command' daemon &
     _NIX_TEST_DAEMON_PID=$!
     export _NIX_TEST_DAEMON_PID
     for ((i = 0; i < 300; i++)); do
@@ -148,7 +148,7 @@ fi
 isDaemonNewer () {
   [[ -n "${NIX_DAEMON_PACKAGE:-}" ]] || return 0
   local requiredVersion="$1"
-  local daemonVersion=$($NIX_DAEMON_PACKAGE/bin/nix-daemon --version | cut -d' ' -f3)
+  local daemonVersion=$($NIX_DAEMON_PACKAGE/bin/nix daemon --version | cut -d' ' -f3)
   [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''$requiredVersion''") -ge 0 ]]
 }
 
diff --git a/tests/functional/nix-daemon-untrusting.sh b/tests/functional/nix-daemon-untrusting.sh
index bcdb70989..c339b5833 100755
--- a/tests/functional/nix-daemon-untrusting.sh
+++ b/tests/functional/nix-daemon-untrusting.sh
@@ -1,3 +1,3 @@
 #!/bin/sh
 
-exec nix-daemon --force-untrusted "$@"
+exec nix daemon --force-untrusted "$@"
diff --git a/tests/functional/store-info.sh b/tests/functional/store-info.sh
index c002e50be..18a8131a9 100644
--- a/tests/functional/store-info.sh
+++ b/tests/functional/store-info.sh
@@ -6,7 +6,7 @@ STORE_INFO_JSON=$(nix store info --json)
 echo "$STORE_INFO" | grep "Store URL: ${NIX_REMOTE}"
 
 if [[ -v NIX_DAEMON_PACKAGE ]] && isDaemonNewer "2.7.0pre20220126"; then
-    DAEMON_VERSION=$($NIX_DAEMON_PACKAGE/bin/nix-daemon --version | cut -d' ' -f3)
+    DAEMON_VERSION=$($NIX_DAEMON_PACKAGE/bin/nix daemon --version | cut -d' ' -f3)
     echo "$STORE_INFO" | grep "Version: $DAEMON_VERSION"
     [[ "$(echo "$STORE_INFO_JSON" | jq -r ".version")" == "$DAEMON_VERSION" ]]
 fi

From ba0087316acc2aba999cabe5e1a159da636b2569 Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.e.helbling@outlook.com>
Date: Mon, 18 Dec 2023 12:59:58 -0800
Subject: [PATCH 256/421] package: don't set sysconfdir in devShells

---
 flake.nix   | 2 +-
 package.nix | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/flake.nix b/flake.nix
index 8c4436729..a8fc105e8 100644
--- a/flake.nix
+++ b/flake.nix
@@ -395,7 +395,7 @@
           stdenvs)));
 
       devShells = let
-        makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; }).overrideAttrs (attrs: {
+        makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: {
           installFlags = "sysconfdir=$(out)/etc";
           shellHook = ''
             PATH=$prefix/bin:$PATH
diff --git a/package.nix b/package.nix
index 370820c40..b5ff45083 100644
--- a/package.nix
+++ b/package.nix
@@ -87,6 +87,9 @@
 , test-daemon ? null
 , test-client ? null
 
+# Avoid setting things that would interfere with a functioning devShell
+, forDevShell ? false
+
 # Not a real argument, just the only way to approximate let-binding some
 # stuff for argument defaults.
 , __forDefaults ? {
@@ -263,13 +266,14 @@ in {
   );
 
   configureFlags = [
-    "--sysconfdir=/etc"
     (lib.enableFeature doBuild "build")
     (lib.enableFeature buildUnitTests "unit-tests")
     (lib.enableFeature doInstallCheck "functional-tests")
     (lib.enableFeature enableInternalAPIDocs "internal-api-docs")
     (lib.enableFeature enableManual "doc-gen")
     (lib.enableFeature installUnitTests "install-unit-tests")
+  ] ++ lib.optionals (!forDevShell) [
+    "--sysconfdir=/etc"
   ] ++ lib.optionals installUnitTests [
     "--with-check-bin-dir=${builtins.placeholder "check"}/bin"
     "--with-check-lib-dir=${builtins.placeholder "check"}/lib"

From 6f4930382bb61f0a9b2a9e5b0080977a4dd03866 Mon Sep 17 00:00:00 2001
From: Rebecca Turner <rbt@sent.as>
Date: Mon, 18 Dec 2023 13:34:08 -0800
Subject: [PATCH 257/421] Document more `Makefile` variables

---
 doc/manual/src/contributing/hacking.md | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index 9478c424d..dce0422dc 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -66,13 +66,24 @@ You can also build Nix for one of the [supported platforms](#platforms).
 
 ## Makefile variables
 
+You may need `profiledir=$out/etc/profile.d` and `sysconfdir=$out/etc` to run
+`make install`.
+
+You may want to set `MAKEFLAGS="-e -j $NIX_BUILD_CORES"` to allow environment
+variables to override `Makefile` variables.
+
 - `ENABLE_BUILD=yes` to enable building the C++ code.
 - `ENABLE_DOC_GEN=yes` to enable building the documentation (manual, man pages, etc.).
+
+  The docs can take a while to build, so you may want to disable this for local development.
 - `ENABLE_FUNCTIONAL_TESTS=yes` to enable building the functional tests.
 - `ENABLE_UNIT_TESTS=yes` to enable building the unit tests.
 - `OPTIMIZE=1` to enable optimizations.
-
-  The docs can take a while to build, so you may want to disable this for local development.
+- `libraries=libutil programs=` to only build a specific library (this will
+  fail in the linking phase if you don't have the other libraries built, but is
+  useful for checking types).
+- `libraries= programs=nix` to only build a specific program (this will not, in
+  general, work, because the programs need the libraries).
 
 ## Building Nix
 

From 0cee56db1ace13a1f4b856c800950b2fb04df993 Mon Sep 17 00:00:00 2001
From: Rebecca Turner <rbt@sent.as>
Date: Mon, 18 Dec 2023 14:04:25 -0800
Subject: [PATCH 258/421] Fix `logging.sh` test on macOS
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

On macOS in the `nix develop` shell, `make
tests/functional/logging.sh.test` errors:

    ++(logging.sh:18) mktemp
    +(logging.sh:18) builder=/var/folders/z5/fclwwdms3r1gq4k4p3pkvvc00000gn/T/tmp.StuabKUhMh
    +(logging.sh:19) echo -e '#!/bin/sh\nmkdir $out'
    +++(logging.sh:22) mktemp -d
    ++(logging.sh:22) nix-build -E 'with import ./config.nix; mkDerivation { name = "fnord"; builder = /var/folders/z5/fclwwdms3r1gq4k4p3pkvvc00000gn/T/tmp.StuabKUhMh; }' --out-link /var/folders/z5/fclwwdms3r1gq4k4p3pkvvc00000gn/T/tmp.oaKcy0NXqC/result
    error:
           … while calling the 'derivationStrict' builtin
             at <nix/derivation-internal.nix>:9:12:
                8|
                9|   strict = derivationStrict drvAttrs;
                 |            ^
               10|

           … while evaluating derivation 'fnord'
             whose name attribute is located at «string»:1:42

           … while evaluating attribute 'args' of derivation 'fnord'
             at /Users/wiggles/nix/tests/functional/config.nix:23:7:
               22|       builder = shell;
               23|       args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" ''
                 |       ^
               24|         if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi;

           error: path '/var' is a symlink
    +(logging.sh:22) outp=
    ++(logging.sh:22) onError
    ++(/Users/wiggles/nix/tests/functional/common/vars-and-functions.sh:237) set +x
    logging.sh: test failed at:
      main in logging.sh:22

This is because `mktemp` returns a path like
`/var/folders/z5/fclwwdms3r1gq4k4p3pkvvc00000gn/T/tmp.qDY24l6bIM`,
where `/var` is a symlink to `/private/var`.

Then, we attempt to use that path as a `builder`, which errors because
symlinks are impure or whatever.

Anyways, we can fix this by using `realpath "$(mktemp)"` instead of
`mktemp` directly.

NB: This error doesn't seem to happen when I run the tests through `nix
flake check`. I'm not sure if Nix does something to `TMP` in that case.
---
 tests/functional/logging.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/functional/logging.sh b/tests/functional/logging.sh
index 1481b9b36..1ccc21d0b 100644
--- a/tests/functional/logging.sh
+++ b/tests/functional/logging.sh
@@ -15,7 +15,7 @@ nix-build dependencies.nix --no-out-link --compress-build-log
 [ "$(nix-store -l $path)" = FOO ]
 
 # test whether empty logs work fine with `nix log`.
-builder="$(mktemp)"
+builder="$(realpath "$(mktemp)")"
 echo -e "#!/bin/sh\nmkdir \$out" > "$builder"
 outp="$(nix-build -E \
     'with import ./config.nix; mkDerivation { name = "fnord"; builder = '"$builder"'; }' \

From 23fb19cb18709ed097274d427a1024ae08789ed3 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 18 Dec 2023 22:36:31 +0000
Subject: [PATCH 259/421] build(deps): bump zeebe-io/backport-action from 2.2.0
 to 2.3.0

Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 2.2.0 to 2.3.0.
- [Release notes](https://github.com/zeebe-io/backport-action/releases)
- [Commits](https://github.com/zeebe-io/backport-action/compare/v2.2.0...v2.3.0)

---
updated-dependencies:
- dependency-name: zeebe-io/backport-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/backport.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
index 85ddcfad3..f003114ba 100644
--- a/.github/workflows/backport.yml
+++ b/.github/workflows/backport.yml
@@ -21,7 +21,7 @@ jobs:
           fetch-depth: 0
       - name: Create backport PRs
         # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v2.2.0
+        uses: zeebe-io/backport-action@v2.3.0
         with:
           # Config README: https://github.com/zeebe-io/backport-action#backport-action
           github_token: ${{ secrets.GITHUB_TOKEN }}

From fa4bbe53e837a138c382468601cd769736f7d1dc Mon Sep 17 00:00:00 2001
From: Mel Zuser <mel.zuser@arista.com>
Date: Mon, 18 Dec 2023 15:02:26 -0800
Subject: [PATCH 260/421] installer: allow overriding of NIX_FIRST_BUILD_ID on
 darwin

because there are often already users in the 300 range and it's painful
to work around.

revives #6466
---
 scripts/install-darwin-multi-user.sh  |  6 ++++--
 scripts/install-multi-user.sh         | 19 +++++++++++++++----
 scripts/install-systemd-multi-user.sh |  4 ++++
 3 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh
index 0326d3415..766f81bde 100644
--- a/scripts/install-darwin-multi-user.sh
+++ b/scripts/install-darwin-multi-user.sh
@@ -3,11 +3,13 @@
 set -eu
 set -o pipefail
 
+# System specific settings
+export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-301}"
+export NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d"
+
 readonly NIX_DAEMON_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist
 # create by default; set 0 to DIY, use a symlink, etc.
 readonly NIX_VOLUME_CREATE=${NIX_VOLUME_CREATE:-1} # now default
-NIX_FIRST_BUILD_UID="301"
-NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d"
 
 # caution: may update times on / if not run as normal non-root user
 read_only_root() {
diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index a08f62333..ad3ee8881 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -25,9 +25,9 @@ readonly RED='\033[31m'
 readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32}
 readonly NIX_BUILD_GROUP_ID="${NIX_BUILD_GROUP_ID:-30000}"
 readonly NIX_BUILD_GROUP_NAME="nixbld"
-# darwin installer needs to override these
-NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-30001}"
-NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d"
+# each system specific installer must set these:
+#   NIX_FIRST_BUILD_UID
+#   NIX_BUILD_USER_NAME_TEMPLATE
 # Please don't change this. We don't support it, because the
 # default shell profile that comes with Nix doesn't support it.
 readonly NIX_ROOT="/nix"
@@ -707,6 +707,12 @@ EOF
     fi
 }
 
+check_required_system_specific_settings() {
+    if [ -z "${NIX_FIRST_BUILD_UID+x}" ] || [ -z "${NIX_BUILD_USER_NAME_TEMPLATE+x}" ]; then
+        failure "Internal error: System specific installer for $(uname) ($1) does not export required settings."
+    fi
+}
+
 welcome_to_nix() {
     local -r NIX_UID_RANGES="${NIX_FIRST_BUILD_UID}..$((NIX_FIRST_BUILD_UID + NIX_USER_COUNT - 1))"
     local -r RANGE_TEXT=$(echo -ne "${BLUE}(uids [${NIX_UID_RANGES}])${ESC}")
@@ -726,7 +732,9 @@ manager. This will happen in a few stages:
    if you are ready to continue.
 
 3. Create the system users ${RANGE_TEXT} and groups ${GROUP_TEXT}
-   that the Nix daemon uses to run builds.
+   that the Nix daemon uses to run builds. To create system users
+   in a different range, exit and run this tool again with
+   NIX_FIRST_BUILD_UID set.
 
 4. Perform the basic installation of the Nix files daemon.
 
@@ -968,13 +976,16 @@ main() {
     if is_os_darwin; then
         # shellcheck source=./install-darwin-multi-user.sh
         . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh"
+        check_required_system_specific_settings "install-darwin-multi-user.sh"
     elif is_os_linux; then
         # shellcheck source=./install-systemd-multi-user.sh
         . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" # most of this works on non-systemd distros also
+        check_required_system_specific_settings "install-systemd-multi-user.sh"
     else
         failure "Sorry, I don't know what to do on $(uname)"
     fi
 
+
     welcome_to_nix
 
     if ! is_root; then
diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh
index 07b34033a..202a9bb54 100755
--- a/scripts/install-systemd-multi-user.sh
+++ b/scripts/install-systemd-multi-user.sh
@@ -3,6 +3,10 @@
 set -eu
 set -o pipefail
 
+# System specific settings
+export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-30001}"
+export NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d"
+
 readonly SERVICE_SRC=/lib/systemd/system/nix-daemon.service
 readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service
 

From 7526b7ded6d5884cefcd4c71e0a33962d883ae78 Mon Sep 17 00:00:00 2001
From: Andrew Marshall <andrew@johnandrewmarshall.com>
Date: Mon, 18 Dec 2023 19:33:20 -0500
Subject: [PATCH 261/421] Allow access to /dev/stderr in Darwin sandbox

We allow /dev/stdout, so why not this? Since it is process-local,
anyway, should not be possible to escape sandbox using it.
---
 src/libstore/build/sandbox-defaults.sb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/build/sandbox-defaults.sb b/src/libstore/build/sandbox-defaults.sb
index 77f013aea..25ec11285 100644
--- a/src/libstore/build/sandbox-defaults.sb
+++ b/src/libstore/build/sandbox-defaults.sb
@@ -68,6 +68,7 @@ R""(
 (allow file*
        (literal "/dev/null")
        (literal "/dev/random")
+       (literal "/dev/stderr")
        (literal "/dev/stdin")
        (literal "/dev/stdout")
        (literal "/dev/tty")

From 0218e4e6c386e4c432520506568420c3cc384e47 Mon Sep 17 00:00:00 2001
From: pennae <github@quasiparticle.net>
Date: Sun, 10 Dec 2023 04:15:51 +0100
Subject: [PATCH 262/421] memset less in addToStoreFromDump
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

resizing a std::string clears the newly added bytes, which is not
necessary here and comes with a ~1.4% slowdown on our test nixos config.

〉 nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'

before:

  Time (mean ± σ):      4.486 s ±  0.003 s    [User: 3.978 s, System: 0.507 s]
  Range (min … max):    4.482 s …  4.492 s    10 runs

after:

  Time (mean ± σ):      4.429 s ±  0.002 s    [User: 3.929 s, System: 0.500 s]
  Range (min … max):    4.427 s …  4.433 s    10 runs
---
 src/libstore/local-store.cc | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 7e82bae28..d903bb061 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -18,6 +18,8 @@
 #include <algorithm>
 #include <cstring>
 
+#include <memory>
+#include <new>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/select.h>
@@ -1130,7 +1132,11 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
        path. */
     bool inMemory = false;
 
-    std::string dump;
+    struct Free {
+        void operator()(void* v) { free(v); }
+    };
+    std::unique_ptr<char, Free> dumpBuffer(nullptr);
+    std::string_view dump;
 
     /* Fill out buffer, and decide whether we are working strictly in
        memory based on whether we break out because the buffer is full
@@ -1139,13 +1145,18 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
         auto oldSize = dump.size();
         constexpr size_t chunkSize = 65536;
         auto want = std::min(chunkSize, settings.narBufferSize - oldSize);
-        dump.resize(oldSize + want);
+        if (auto tmp = realloc(dumpBuffer.get(), oldSize + want)) {
+            dumpBuffer.release();
+            dumpBuffer.reset((char*) tmp);
+        } else {
+            throw std::bad_alloc();
+        }
         auto got = 0;
         Finally cleanup([&]() {
-            dump.resize(oldSize + got);
+            dump = {dumpBuffer.get(), dump.size() + got};
         });
         try {
-            got = source.read(dump.data() + oldSize, want);
+            got = source.read(dumpBuffer.get() + oldSize, want);
         } catch (EndOfFile &) {
             inMemory = true;
             break;
@@ -1171,7 +1182,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
         else
             writeFile(tempPath, bothSource);
 
-        dump.clear();
+        dumpBuffer.reset();
+        dump = {};
     }
 
     auto [hash, size] = hashSink->finish();

From 78353deb028fcc700776db9d92dcae45d68fb85f Mon Sep 17 00:00:00 2001
From: pennae <github@quasiparticle.net>
Date: Sun, 10 Dec 2023 08:24:45 +0100
Subject: [PATCH 263/421] encode black holes as tApp values
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

checking for isBlackhole in the forceValue hot path is rather more
expensive than necessary, and with a little bit of trickery we can move
such handling into the isApp case. small performance benefit, but under
some circumstances we've seen 2% improvement as well.

〉 nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'

before:

  Time (mean ± σ):      4.429 s ±  0.002 s    [User: 3.929 s, System: 0.500 s]
  Range (min … max):    4.427 s …  4.433 s    10 runs

after:

  Time (mean ± σ):      4.396 s ±  0.002 s    [User: 3.894 s, System: 0.501 s]
  Range (min … max):    4.393 s …  4.399 s    10 runs
---
 src/libexpr/eval-inline.hh | 13 +++++++----
 src/libexpr/eval.cc        | 44 +++++++++++++++++++++-----------------
 src/libexpr/eval.hh        |  8 +++++++
 src/libexpr/nixexpr.hh     |  7 ++++++
 src/libexpr/primops.cc     | 23 ++++++++++++++++++++
 src/libexpr/primops.hh     |  6 ++++++
 src/libexpr/value.hh       | 24 ++++++++++++++-------
 7 files changed, 93 insertions(+), 32 deletions(-)

diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index c37b1d62b..9d08f1938 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -104,11 +104,16 @@ void EvalState::forceValue(Value & v, Callable getPos)
         }
     }
     else if (v.isApp()) {
-        PosIdx pos = getPos();
-        callFunction(*v.app.left, *v.app.right, v, pos);
+        try {
+            callFunction(*v.app.left, *v.app.right, v, noPos);
+        } catch (InfiniteRecursionError & e) {
+            // only one black hole can *throw* in any given eval stack so we need not
+            // check whether the position is set already.
+            if (v.isBlackhole())
+                e.err.errPos = positions[getPos()];
+            throw;
+        }
     }
-    else if (v.isBlackhole())
-        error("infinite recursion encountered").atPos(getPos()).template debugThrow<EvalError>();
 }
 
 
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 9e494148e..71c151f96 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -162,7 +162,17 @@ void Value::print(const SymbolTable &symbols, std::ostream &str,
         break;
     case tThunk:
     case tApp:
-        str << "<CODE>";
+        if (!isBlackhole()) {
+            str << "<CODE>";
+        } else {
+            // Although we know for sure that it's going to be an infinite recursion
+            // when this value is accessed _in the current context_, it's likely
+            // that the user will misinterpret a simpler «infinite recursion» output
+            // as a definitive statement about the value, while in fact it may be
+            // a valid value after `builtins.trace` and perhaps some other steps
+            // have completed.
+            str << "«potential infinite recursion»";
+        }
         break;
     case tLambda:
         str << "<LAMBDA>";
@@ -179,15 +189,6 @@ void Value::print(const SymbolTable &symbols, std::ostream &str,
     case tFloat:
         str << fpoint;
         break;
-    case tBlackhole:
-        // Although we know for sure that it's going to be an infinite recursion
-        // when this value is accessed _in the current context_, it's likely
-        // that the user will misinterpret a simpler «infinite recursion» output
-        // as a definitive statement about the value, while in fact it may be
-        // a valid value after `builtins.trace` and perhaps some other steps
-        // have completed.
-        str << "«potential infinite recursion»";
-        break;
     default:
         printError("Nix evaluator internal error: Value::print(): invalid value type %1%", internalType);
         abort();
@@ -256,8 +257,7 @@ std::string showType(const Value & v)
             return fmt("the partially applied built-in function '%s'", std::string(getPrimOp(v)->primOp->name));
         case tExternal: return v.external->showType();
         case tThunk: return "a thunk";
-        case tApp: return "a function application";
-        case tBlackhole: return "a black hole";
+        case tApp: return v.isBlackhole() ? "a black hole" : "a function application";
     default:
         return std::string(showType(v.type()));
     }
@@ -1621,15 +1621,17 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 return;
             } else {
                 /* We have all the arguments, so call the primop. */
-                auto name = vCur.primOp->name;
+                auto * fn = vCur.primOp;
 
                 nrPrimOpCalls++;
-                if (countCalls) primOpCalls[name]++;
+                // This will count black holes, but that's ok, because unrecoverable errors are rare.
+                if (countCalls) primOpCalls[fn->name]++;
 
                 try {
-                    vCur.primOp->fun(*this, vCur.determinePos(noPos), args, vCur);
+                    fn->fun(*this, vCur.determinePos(noPos), args, vCur);
                 } catch (Error & e) {
-                    addErrorTrace(e, pos, "while calling the '%1%' builtin", name);
+                    if (!fn->hideInDiagnostics)
+                        addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
                     throw;
                 }
 
@@ -1666,18 +1668,20 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 for (size_t i = 0; i < argsLeft; ++i)
                     vArgs[argsDone + i] = args[i];
 
-                auto name = primOp->primOp->name;
+                auto fn = primOp->primOp;
                 nrPrimOpCalls++;
-                if (countCalls) primOpCalls[name]++;
+                // This will count black holes, but that's ok, because unrecoverable errors are rare.
+                if (countCalls) primOpCalls[fn->name]++;
 
                 try {
                     // TODO:
                     // 1. Unify this and above code. Heavily redundant.
                     // 2. Create a fake env (arg1, arg2, etc.) and a fake expr (arg1: arg2: etc: builtins.name arg1 arg2 etc)
                     //    so the debugger allows to inspect the wrong parameters passed to the builtin.
-                    primOp->primOp->fun(*this, vCur.determinePos(noPos), vArgs, vCur);
+                    fn->fun(*this, vCur.determinePos(noPos), vArgs, vCur);
                 } catch (Error & e) {
-                    addErrorTrace(e, pos, "while calling the '%1%' builtin", name);
+                    if (!fn->hideInDiagnostics)
+                        addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
                     throw;
                 }
 
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index f3f6d35b9..e5e401ab6 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -77,6 +77,14 @@ struct PrimOp
      */
     std::optional<ExperimentalFeature> experimentalFeature;
 
+    /**
+     * Whether to hide this primop in diagnostics.
+     *
+     * Used to hide the fact that black holes are primop applications from
+     * stack traces.
+     */
+    bool hideInDiagnostics;
+
     /**
      * Validity check to be performed by functions that introduce primops,
      * such as RegisterPrimOp() and Value::mkPrimOp().
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index 020286815..cf6fd1a8d 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -21,6 +21,13 @@ MakeError(TypeError, EvalError);
 MakeError(UndefinedVarError, Error);
 MakeError(MissingArgumentError, EvalError);
 
+class InfiniteRecursionError : public EvalError
+{
+    friend class EvalState;
+public:
+    using EvalError::EvalError;
+};
+
 /**
  * Position objects.
  */
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 89d5492da..d46eccd34 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -4263,6 +4263,29 @@ static RegisterPrimOp primop_splitVersion({
 });
 
 
+static void prim_blackHoleFn(EvalState & state, const PosIdx pos, Value * * args, Value & v)
+{
+    state.error("infinite recursion encountered")
+        .debugThrow<InfiniteRecursionError>();
+}
+
+static PrimOp primop_blackHole = {
+    .name = "«blackHole»",
+    .args = {},
+    .fun = prim_blackHoleFn,
+    .hideInDiagnostics = true,
+};
+
+static Value makeBlackHole()
+{
+    Value v;
+    v.mkPrimOp(&primop_blackHole);
+    return v;
+}
+
+Value prim_blackHole = makeBlackHole();
+
+
 /*************************************************************
  * Primop registration
  *************************************************************/
diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh
index 45486608f..244eada86 100644
--- a/src/libexpr/primops.hh
+++ b/src/libexpr/primops.hh
@@ -51,4 +51,10 @@ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Valu
  */
 void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v);
 
+/**
+ *  Placeholder value for black holes, used to represent black holes as
+ *  applications of this value to the evaluated thunks.
+ */
+extern Value prim_blackHole;
+
 }
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 30b3d4934..52cd0f901 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -32,7 +32,6 @@ typedef enum {
     tThunk,
     tApp,
     tLambda,
-    tBlackhole,
     tPrimOp,
     tPrimOpApp,
     tExternal,
@@ -151,7 +150,7 @@ public:
     // type() == nThunk
     inline bool isThunk() const { return internalType == tThunk; };
     inline bool isApp() const { return internalType == tApp; };
-    inline bool isBlackhole() const { return internalType == tBlackhole; };
+    inline bool isBlackhole() const;
 
     // type() == nFunction
     inline bool isLambda() const { return internalType == tLambda; };
@@ -248,7 +247,7 @@ public:
             case tLambda: case tPrimOp: case tPrimOpApp: return nFunction;
             case tExternal: return nExternal;
             case tFloat: return nFloat;
-            case tThunk: case tApp: case tBlackhole: return nThunk;
+            case tThunk: case tApp: return nThunk;
         }
         if (invalidIsThunk)
             return nThunk;
@@ -356,11 +355,7 @@ public:
         lambda.fun = f;
     }
 
-    inline void mkBlackhole()
-    {
-        internalType = tBlackhole;
-        // Value will be overridden anyways
-    }
+    inline void mkBlackhole();
 
     void mkPrimOp(PrimOp * p);
 
@@ -447,6 +442,19 @@ public:
 };
 
 
+extern Value prim_blackHole;
+
+inline bool Value::isBlackhole() const
+{
+    return internalType == tApp && app.left == &prim_blackHole;
+}
+
+inline void Value::mkBlackhole()
+{
+    mkApp(&prim_blackHole, &prim_blackHole);
+}
+
+
 #if HAVE_BOEHMGC
 typedef std::vector<Value *, traceable_allocator<Value *>> ValueVector;
 typedef std::map<Symbol, Value *, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, Value *>>> ValueMap;

From 74c134914c747b1df6385cab5d2298f66a87b61f Mon Sep 17 00:00:00 2001
From: pennae <github@quasiparticle.net>
Date: Sun, 10 Dec 2023 09:25:20 +0100
Subject: [PATCH 264/421] compare string values with strcmp

string_view()ification calls strlen() first, which we don't need here.
---
 src/libexpr/eval.cc    | 2 +-
 src/libexpr/primops.cc | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 71c151f96..8e89ddcf1 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -2436,7 +2436,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
             return v1.boolean == v2.boolean;
 
         case nString:
-            return v1.string_view().compare(v2.string_view()) == 0;
+            return strcmp(v1.c_str(), v2.c_str()) == 0;
 
         case nPath:
             return
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index d46eccd34..b7e903667 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -586,7 +586,7 @@ struct CompareValues
                 case nFloat:
                     return v1->fpoint < v2->fpoint;
                 case nString:
-                    return v1->string_view().compare(v2->string_view()) < 0;
+                    return strcmp(v1->c_str(), v2->c_str()) < 0;
                 case nPath:
                     // Note: we don't take the accessor into account
                     // since it's not obvious how to compare them in a
@@ -2401,7 +2401,7 @@ static void prim_attrNames(EvalState & state, const PosIdx pos, Value * * args,
         (v.listElems()[n++] = state.allocValue())->mkString(state.symbols[i.name]);
 
     std::sort(v.listElems(), v.listElems() + n,
-              [](Value * v1, Value * v2) { return v1->string_view().compare(v2->string_view()) < 0; });
+              [](Value * v1, Value * v2) { return strcmp(v1->c_str(), v2->c_str()) < 0; });
 }
 
 static RegisterPrimOp primop_attrNames({

From cc4038d54177c944340607c7d141680e66ff92a7 Mon Sep 17 00:00:00 2001
From: pennae <github@quasiparticle.net>
Date: Sun, 10 Dec 2023 09:49:38 +0100
Subject: [PATCH 265/421] use std::tie() for macro-generated operators
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

as written the comparisons generate copies, even though it looks as
though they shouldn't.

before:

  Time (mean ± σ):      4.396 s ±  0.002 s    [User: 3.894 s, System: 0.501 s]
  Range (min … max):    4.393 s …  4.399 s    10 runs

after:

  Time (mean ± σ):      4.260 s ±  0.003 s    [User: 3.754 s, System: 0.505 s]
  Range (min … max):    4.257 s …  4.266 s    10 runs
---
 src/libcmd/built-path.cc     | 4 ++--
 src/libstore/derived-path.cc | 8 ++------
 src/libutil/comparator.hh    | 4 ++--
 3 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc
index 8e2efc7c3..c5eb93c5d 100644
--- a/src/libcmd/built-path.cc
+++ b/src/libcmd/built-path.cc
@@ -12,9 +12,9 @@ namespace nix {
     bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \
     { \
         const MY_TYPE* me = this; \
-        auto fields1 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
+        auto fields1 = std::tie(*me->drvPath, me->FIELD); \
         me = &other; \
-        auto fields2 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
+        auto fields2 = std::tie(*me->drvPath, me->FIELD); \
         return fields1 COMPARATOR fields2; \
     }
 #define CMP(CHILD_TYPE, MY_TYPE, FIELD) \
diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc
index 3105dbc93..a7b404321 100644
--- a/src/libstore/derived-path.cc
+++ b/src/libstore/derived-path.cc
@@ -12,9 +12,9 @@ namespace nix {
     bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \
     { \
         const MY_TYPE* me = this; \
-        auto fields1 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
+        auto fields1 = std::tie(*me->drvPath, me->FIELD); \
         me = &other; \
-        auto fields2 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
+        auto fields2 = std::tie(*me->drvPath, me->FIELD); \
         return fields1 COMPARATOR fields2; \
     }
 #define CMP(CHILD_TYPE, MY_TYPE, FIELD) \
@@ -22,13 +22,9 @@ namespace nix {
     CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \
     CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <)
 
-#define FIELD_TYPE std::string
 CMP(SingleDerivedPath, SingleDerivedPathBuilt, output)
-#undef FIELD_TYPE
 
-#define FIELD_TYPE OutputsSpec
 CMP(SingleDerivedPath, DerivedPathBuilt, outputs)
-#undef FIELD_TYPE
 
 #undef CMP
 #undef CMP_ONE
diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh
index a4d20a675..cbc2bb4fd 100644
--- a/src/libutil/comparator.hh
+++ b/src/libutil/comparator.hh
@@ -13,9 +13,9 @@
 #define GENERATE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE, ...) \
     PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const { \
       __VA_OPT__(const MY_TYPE * me = this;) \
-      auto fields1 = std::make_tuple( __VA_ARGS__ ); \
+      auto fields1 = std::tie( __VA_ARGS__ ); \
       __VA_OPT__(me = &other;) \
-      auto fields2 = std::make_tuple( __VA_ARGS__ ); \
+      auto fields2 = std::tie( __VA_ARGS__ ); \
       return fields1 COMPARATOR fields2; \
     }
 #define GENERATE_EQUAL(prefix, qualification, my_type, args...) \

From 2e0321912a9efa352160eb1e57e6b7b88e517d0d Mon Sep 17 00:00:00 2001
From: pennae <github@quasiparticle.net>
Date: Sun, 10 Dec 2023 12:59:51 +0100
Subject: [PATCH 266/421] use aligned flex tables
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

~2% speedup on parsing without eval, less (but still significant) on
system eval. having flex generate faster parsers leads to very strange
misparses. maybe re2c is worth investigating.

before:

  Time (mean ± σ):      4.260 s ±  0.003 s    [User: 3.754 s, System: 0.505 s]
  Range (min … max):    4.257 s …  4.266 s    10 runs

after:

  Time (mean ± σ):      4.231 s ±  0.004 s    [User: 3.725 s, System: 0.504 s]
  Range (min … max):    4.226 s …  4.240 s    10 runs
---
 src/libexpr/lexer.l | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index a3a8608d9..9a35dd594 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -1,4 +1,5 @@
 %option reentrant bison-bridge bison-locations
+%option align
 %option noyywrap
 %option never-interactive
 %option stack

From b78e77b34c14b0f127b22e252309527e84967dcc Mon Sep 17 00:00:00 2001
From: pennae <github@quasiparticle.net>
Date: Sun, 10 Dec 2023 13:00:18 +0100
Subject: [PATCH 267/421] use custom location type in the parser
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

~1% parser speedup from not using TLS indirections, less on system eval.
this could have also gone in flex yyextra data, but that's significantly
slower for some reason (albeit still faster than thread locals).

before:

  Time (mean ± σ):      4.231 s ±  0.004 s    [User: 3.725 s, System: 0.504 s]
  Range (min … max):    4.226 s …  4.240 s    10 runs

after:

  Time (mean ± σ):      4.224 s ±  0.005 s    [User: 3.711 s, System: 0.512 s]
  Range (min … max):    4.218 s …  4.234 s    10 runs
---
 src/libexpr/lexer.l  |  9 +++------
 src/libexpr/parser.y | 25 +++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index 9a35dd594..df2cbd06f 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -36,9 +36,6 @@ static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data)
 
 #define CUR_POS makeCurPos(*yylloc, data)
 
-// backup to recover from yyless(0)
-thread_local YYLTYPE prev_yylloc;
-
 static void initLoc(YYLTYPE * loc)
 {
     loc->first_line = loc->last_line = 1;
@@ -47,7 +44,7 @@ static void initLoc(YYLTYPE * loc)
 
 static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
 {
-    prev_yylloc = *loc;
+    loc->stash();
 
     loc->first_line = loc->last_line;
     loc->first_column = loc->last_column;
@@ -231,7 +228,7 @@ or          { return OR_KW; }
 {HPATH_START}\$\{ {
   PUSH_STATE(PATH_START);
   yyless(0);
-  *yylloc = prev_yylloc;
+  yylloc->unstash();
 }
 
 <PATH_START>{PATH_SEG} {
@@ -287,7 +284,7 @@ or          { return OR_KW; }
      context (it may be ')', ';', or something of that sort) */
   POP_STATE();
   yyless(0);
-  *yylloc = prev_yylloc;
+  yylloc->unstash();
   return PATH_END;
 }
 
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index 16ad8af2e..b331776f0 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -28,6 +28,31 @@
 
 namespace nix {
 
+#define YYLTYPE ::nix::ParserLocation
+    struct ParserLocation
+    {
+        int first_line, first_column;
+        int last_line, last_column;
+
+        // backup to recover from yyless(0)
+        int stashed_first_line, stashed_first_column;
+        int stashed_last_line, stashed_last_column;
+
+        void stash() {
+          stashed_first_line = first_line;
+          stashed_first_column = first_column;
+          stashed_last_line = last_line;
+          stashed_last_column = last_column;
+        }
+
+        void unstash() {
+          first_line = stashed_first_line;
+          first_column = stashed_first_column;
+          last_line = stashed_last_line;
+          last_column = stashed_last_column;
+        }
+    };
+
     struct ParseData
     {
         EvalState & state;

From f9aee2f2c41652b3b76d16a874fdded4e6d28d92 Mon Sep 17 00:00:00 2001
From: pennae <github@quasiparticle.net>
Date: Sun, 10 Dec 2023 10:34:55 +0100
Subject: [PATCH 268/421] don't malloc/memset posix accessor buffer

it's relatively small and fits on the stack nicely, and we don't need it
initialized either.
---
 src/libutil/posix-source-accessor.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc
index 15ff76e59..5f26fa67b 100644
--- a/src/libutil/posix-source-accessor.cc
+++ b/src/libutil/posix-source-accessor.cc
@@ -25,7 +25,7 @@ void PosixSourceAccessor::readFile(
 
     off_t left = st.st_size;
 
-    std::vector<unsigned char> buf(64 * 1024);
+    std::array<unsigned char, 64 * 1024> buf;
     while (left) {
         checkInterrupt();
         ssize_t rd = read(fd.get(), buf.data(), (size_t) std::min(left, (off_t) buf.size()));

From 69ed4aee612e247f2d6ebbb44aba743c4282e00e Mon Sep 17 00:00:00 2001
From: pennae <github@quasiparticle.net>
Date: Mon, 11 Dec 2023 15:48:24 +0100
Subject: [PATCH 269/421] remove lazy-pos forceValue

almost all uses of this are interactive, except for deepSeq. deepSeq is
going to be expensive and rare enough to not care much about, and
Value::determinePos should usually be cheap enough to not be too much of
a burden in any case.
---
 src/libcmd/installable-flake.cc        |  2 +-
 src/libcmd/repl.cc                     |  4 ++--
 src/libexpr/eval-inline.hh             | 10 +---------
 src/libexpr/eval.cc                    |  2 +-
 src/libexpr/eval.hh                    |  3 ---
 src/libexpr/get-drvs.cc                |  4 ++--
 src/nix-build/nix-build.cc             |  2 +-
 src/nix-env/user-env.cc                |  2 +-
 src/nix-instantiate/nix-instantiate.cc |  2 +-
 9 files changed, 10 insertions(+), 21 deletions(-)

diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc
index 2f428cb7e..ddec7537b 100644
--- a/src/libcmd/installable-flake.cc
+++ b/src/libcmd/installable-flake.cc
@@ -52,7 +52,7 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked
     auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
     assert(aOutputs);
 
-    state.forceValue(*aOutputs->value, [&]() { return aOutputs->value->determinePos(noPos); });
+    state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos));
 
     return aOutputs->value;
 }
diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc
index 0986296ad..97d709ff4 100644
--- a/src/libcmd/repl.cc
+++ b/src/libcmd/repl.cc
@@ -888,7 +888,7 @@ void NixRepl::evalString(std::string s, Value & v)
 {
     Expr * e = parseString(s);
     e->eval(*state, *env, v);
-    state->forceValue(v, [&]() { return v.determinePos(noPos); });
+    state->forceValue(v, v.determinePos(noPos));
 }
 
 
@@ -907,7 +907,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
     str.flush();
     checkInterrupt();
 
-    state->forceValue(v, [&]() { return v.determinePos(noPos); });
+    state->forceValue(v, v.determinePos(noPos));
 
     switch (v.type()) {
 
diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index 9d08f1938..8a9ebb77a 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -81,15 +81,7 @@ Env & EvalState::allocEnv(size_t size)
 }
 
 
-[[gnu::always_inline]]
 void EvalState::forceValue(Value & v, const PosIdx pos)
-{
-    forceValue(v, [&]() { return pos; });
-}
-
-
-template<typename Callable>
-void EvalState::forceValue(Value & v, Callable getPos)
 {
     if (v.isThunk()) {
         Env * env = v.thunk.env;
@@ -110,7 +102,7 @@ void EvalState::forceValue(Value & v, Callable getPos)
             // only one black hole can *throw* in any given eval stack so we need not
             // check whether the position is set already.
             if (v.isBlackhole())
-                e.err.errPos = positions[getPos()];
+                e.err.errPos = positions[pos];
             throw;
         }
     }
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 8e89ddcf1..4dc5af97a 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -2044,7 +2044,7 @@ void EvalState::forceValueDeep(Value & v)
     recurse = [&](Value & v) {
         if (!seen.insert(&v).second) return;
 
-        forceValue(v, [&]() { return v.determinePos(noPos); });
+        forceValue(v, v.determinePos(noPos));
 
         if (v.type() == nAttrs) {
             for (auto & i : *v.attrs)
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index e5e401ab6..4c7ea1d98 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -473,9 +473,6 @@ public:
      */
     inline void forceValue(Value & v, const PosIdx pos);
 
-    template <typename Callable>
-    inline void forceValue(Value & v, Callable getPos);
-
     /**
      * Force a value, then recursively force list elements and
      * attributes.
diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index d4e946d81..a6441871c 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -198,7 +198,7 @@ StringSet DrvInfo::queryMetaNames()
 
 bool DrvInfo::checkMeta(Value & v)
 {
-    state->forceValue(v, [&]() { return v.determinePos(noPos); });
+    state->forceValue(v, v.determinePos(noPos));
     if (v.type() == nList) {
         for (auto elem : v.listItems())
             if (!checkMeta(*elem)) return false;
@@ -304,7 +304,7 @@ static bool getDerivation(EvalState & state, Value & v,
     bool ignoreAssertionFailures)
 {
     try {
-        state.forceValue(v, [&]() { return v.determinePos(noPos); });
+        state.forceValue(v, v.determinePos(noPos));
         if (!state.isDerivation(v)) return true;
 
         /* Remove spurious duplicates (e.g., a set like `rec { x =
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 01da028d8..4465e2f90 100644
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -350,7 +350,7 @@ static void main_nix_build(int argc, char * * argv)
                 takesNixShellAttr(vRoot) ? *autoArgsWithInNixShell : *autoArgs,
                 vRoot
             ).first);
-            state->forceValue(v, [&]() { return v.determinePos(noPos); });
+            state->forceValue(v, v.determinePos(noPos));
             getDerivations(
                 *state,
                 v,
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index 34f6bd005..fe5b89b3f 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -128,7 +128,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
 
     /* Evaluate it. */
     debug("evaluating user environment builder");
-    state.forceValue(topLevel, [&]() { return topLevel.determinePos(noPos); });
+    state.forceValue(topLevel, topLevel.determinePos(noPos));
     NixStringContext context;
     Attr & aDrvPath(*topLevel.attrs->find(state.sDrvPath));
     auto topLevelDrv = state.coerceToStorePath(aDrvPath.pos, *aDrvPath.value, context, "");
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index 86b9be17d..ab590b3a6 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -40,7 +40,7 @@ void processExpr(EvalState & state, const Strings & attrPaths,
 
     for (auto & i : attrPaths) {
         Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot).first);
-        state.forceValue(v, [&]() { return v.determinePos(noPos); });
+        state.forceValue(v, v.determinePos(noPos));
 
         NixStringContext context;
         if (evalOnly) {

From f9db4de0f3758e0f730a5d98348e7cc40082104a Mon Sep 17 00:00:00 2001
From: pennae <github@quasiparticle.net>
Date: Mon, 11 Dec 2023 15:54:16 +0100
Subject: [PATCH 270/421] force-inline forceValue
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

forceValue is extremely hot. interestingly adding likeliness annotations
to the branches does not seem to make a difference.

before:

  Time (mean ± σ):      4.224 s ±  0.005 s    [User: 3.711 s, System: 0.512 s]
  Range (min … max):    4.218 s …  4.234 s    10 runs

after:

  Time (mean ± σ):      4.140 s ±  0.009 s    [User: 3.647 s, System: 0.492 s]
  Range (min … max):    4.130 s …  4.152 s    10 runs
---
 src/libexpr/eval-inline.hh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index 8a9ebb77a..d48871628 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -81,6 +81,7 @@ Env & EvalState::allocEnv(size_t size)
 }
 
 
+[[gnu::always_inline]]
 void EvalState::forceValue(Value & v, const PosIdx pos)
 {
     if (v.isThunk()) {

From 2b0e95e7aabd075f95cbfb1607330b2284b01918 Mon Sep 17 00:00:00 2001
From: pennae <github@quasiparticle.net>
Date: Mon, 11 Dec 2023 16:23:08 +0100
Subject: [PATCH 271/421] use singleton expr to generate black hole errors

this also reduces forceValue code size and removes the need for
hideInDiagnostics. coopting thunk forcing like this has the additional
benefit of clarifying how these errors can happen in the first place.
---
 src/libexpr/eval-inline.hh | 14 +++-----------
 src/libexpr/eval.cc        | 35 +++++++++++++++++++++++++++--------
 src/libexpr/eval.hh        | 10 ++--------
 src/libexpr/nixexpr.cc     |  2 ++
 src/libexpr/nixexpr.hh     | 10 ++++++++++
 src/libexpr/primops.cc     | 23 -----------------------
 src/libexpr/primops.hh     |  6 ------
 src/libexpr/value.hh       | 12 +++++++-----
 8 files changed, 51 insertions(+), 61 deletions(-)

diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index d48871628..52aa75b5f 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -93,20 +93,12 @@ void EvalState::forceValue(Value & v, const PosIdx pos)
             expr->eval(*this, *env, v);
         } catch (...) {
             v.mkThunk(env, expr);
+            tryFixupBlackHolePos(v, pos);
             throw;
         }
     }
-    else if (v.isApp()) {
-        try {
-            callFunction(*v.app.left, *v.app.right, v, noPos);
-        } catch (InfiniteRecursionError & e) {
-            // only one black hole can *throw* in any given eval stack so we need not
-            // check whether the position is set already.
-            if (v.isBlackhole())
-                e.err.errPos = positions[pos];
-            throw;
-        }
-    }
+    else if (v.isApp())
+        callFunction(*v.app.left, *v.app.right, v, pos);
 }
 
 
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 4dc5af97a..0c35b3713 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -256,8 +256,8 @@ std::string showType(const Value & v)
         case tPrimOpApp:
             return fmt("the partially applied built-in function '%s'", std::string(getPrimOp(v)->primOp->name));
         case tExternal: return v.external->showType();
-        case tThunk: return "a thunk";
-        case tApp: return v.isBlackhole() ? "a black hole" : "a function application";
+        case tThunk: return v.isBlackhole() ? "a black hole" : "a thunk";
+        case tApp: return "a function application";
     default:
         return std::string(showType(v.type()));
     }
@@ -1624,14 +1624,12 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 auto * fn = vCur.primOp;
 
                 nrPrimOpCalls++;
-                // This will count black holes, but that's ok, because unrecoverable errors are rare.
                 if (countCalls) primOpCalls[fn->name]++;
 
                 try {
                     fn->fun(*this, vCur.determinePos(noPos), args, vCur);
                 } catch (Error & e) {
-                    if (!fn->hideInDiagnostics)
-                        addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
+                    addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
                     throw;
                 }
 
@@ -1670,7 +1668,6 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
 
                 auto fn = primOp->primOp;
                 nrPrimOpCalls++;
-                // This will count black holes, but that's ok, because unrecoverable errors are rare.
                 if (countCalls) primOpCalls[fn->name]++;
 
                 try {
@@ -1680,8 +1677,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                     //    so the debugger allows to inspect the wrong parameters passed to the builtin.
                     fn->fun(*this, vCur.determinePos(noPos), vArgs, vCur);
                 } catch (Error & e) {
-                    if (!fn->hideInDiagnostics)
-                        addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
+                    addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
                     throw;
                 }
 
@@ -2035,6 +2031,29 @@ void ExprPos::eval(EvalState & state, Env & env, Value & v)
 }
 
 
+void ExprBlackHole::eval(EvalState & state, Env & env, Value & v)
+{
+    state.error("infinite recursion encountered")
+        .debugThrow<InfiniteRecursionError>();
+}
+
+// always force this to be separate, otherwise forceValue may inline it and take
+// a massive perf hit
+[[gnu::noinline]]
+void EvalState::tryFixupBlackHolePos(Value & v, PosIdx pos)
+{
+    if (!v.isBlackhole())
+        return;
+    auto e = std::current_exception();
+    try {
+        std::rethrow_exception(e);
+    } catch (InfiniteRecursionError & e) {
+        e.err.errPos = positions[pos];
+    } catch (...) {
+    }
+}
+
+
 void EvalState::forceValueDeep(Value & v)
 {
     std::set<const Value *> seen;
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 4c7ea1d98..56bc5e48f 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -77,14 +77,6 @@ struct PrimOp
      */
     std::optional<ExperimentalFeature> experimentalFeature;
 
-    /**
-     * Whether to hide this primop in diagnostics.
-     *
-     * Used to hide the fact that black holes are primop applications from
-     * stack traces.
-     */
-    bool hideInDiagnostics;
-
     /**
      * Validity check to be performed by functions that introduce primops,
      * such as RegisterPrimOp() and Value::mkPrimOp().
@@ -473,6 +465,8 @@ public:
      */
     inline void forceValue(Value & v, const PosIdx pos);
 
+    void tryFixupBlackHolePos(Value & v, PosIdx pos);
+
     /**
      * Force a value, then recursively force list elements and
      * attributes.
diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc
index 22be8e68c..84860b30f 100644
--- a/src/libexpr/nixexpr.cc
+++ b/src/libexpr/nixexpr.cc
@@ -9,6 +9,8 @@
 
 namespace nix {
 
+ExprBlackHole eBlackHole;
+
 struct PosAdapter : AbstractPos
 {
     Pos::Origin origin;
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index cf6fd1a8d..1e57fec7a 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -462,6 +462,16 @@ struct ExprPos : Expr
     COMMON_METHODS
 };
 
+/* only used to mark thunks as black holes. */
+struct ExprBlackHole : Expr
+{
+    void show(const SymbolTable & symbols, std::ostream & str) const override {}
+    void eval(EvalState & state, Env & env, Value & v) override;
+    void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) override {}
+};
+
+extern ExprBlackHole eBlackHole;
+
 
 /* Static environments are used to map variable names onto (level,
    displacement) pairs used to obtain the value of the variable at
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index b7e903667..2a71747a0 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -4263,29 +4263,6 @@ static RegisterPrimOp primop_splitVersion({
 });
 
 
-static void prim_blackHoleFn(EvalState & state, const PosIdx pos, Value * * args, Value & v)
-{
-    state.error("infinite recursion encountered")
-        .debugThrow<InfiniteRecursionError>();
-}
-
-static PrimOp primop_blackHole = {
-    .name = "«blackHole»",
-    .args = {},
-    .fun = prim_blackHoleFn,
-    .hideInDiagnostics = true,
-};
-
-static Value makeBlackHole()
-{
-    Value v;
-    v.mkPrimOp(&primop_blackHole);
-    return v;
-}
-
-Value prim_blackHole = makeBlackHole();
-
-
 /*************************************************************
  * Primop registration
  *************************************************************/
diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh
index 244eada86..45486608f 100644
--- a/src/libexpr/primops.hh
+++ b/src/libexpr/primops.hh
@@ -51,10 +51,4 @@ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Valu
  */
 void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v);
 
-/**
- *  Placeholder value for black holes, used to represent black holes as
- *  applications of this value to the evaluated thunks.
- */
-extern Value prim_blackHole;
-
 }
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 52cd0f901..d9860e921 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -61,6 +61,7 @@ class Bindings;
 struct Env;
 struct Expr;
 struct ExprLambda;
+struct ExprBlackHole;
 struct PrimOp;
 class Symbol;
 class PosIdx;
@@ -442,16 +443,17 @@ public:
 };
 
 
-extern Value prim_blackHole;
+extern ExprBlackHole eBlackHole;
 
-inline bool Value::isBlackhole() const
+bool Value::isBlackhole() const
 {
-    return internalType == tApp && app.left == &prim_blackHole;
+    return internalType == tThunk && thunk.expr == (Expr*) &eBlackHole;
 }
 
-inline void Value::mkBlackhole()
+void Value::mkBlackhole()
 {
-    mkApp(&prim_blackHole, &prim_blackHole);
+    internalType = tThunk;
+    thunk.expr = (Expr*) &eBlackHole;
 }
 
 

From 26d60b837ca84856ceef18627b2354d26f002eb1 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 20 Dec 2023 03:23:49 -0500
Subject: [PATCH 272/421] Move down fallback targets in `Makefile`

This ensures `lib.mk` still defines `default` as the first target. This
fixes some builds.
---
 Makefile | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/Makefile b/Makefile
index c62216df8..1fdb6e897 100644
--- a/Makefile
+++ b/Makefile
@@ -32,11 +32,6 @@ makefiles += \
   tests/unit/libstore-support/local.mk \
   tests/unit/libexpr/local.mk \
   tests/unit/libexpr-support/local.mk
-else
-.PHONY: check
-check:
-	@echo "Unit tests are disabled. Configure without '--disable-unit-tests', or avoid calling 'make check'."
-	@exit 1
 endif
 
 ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes)
@@ -46,11 +41,6 @@ makefiles += \
   tests/functional/dyn-drv/local.mk \
   tests/functional/test-libstoreconsumer/local.mk \
   tests/functional/plugins/local.mk
-else
-.PHONY: installcheck
-installcheck:
-	@echo "Functional tests are disabled. Configure without '--disable-functional-tests', or avoid calling 'make installcheck'."
-	@exit 1
 endif
 
 OPTIMIZE = 1
@@ -64,9 +54,25 @@ endif
 
 include mk/lib.mk
 
+# Must be included after `mk/lib.mk` so isn't the default target.
+ifneq ($(ENABLE_UNIT_TESTS), yes)
+.PHONY: check
+check:
+	@echo "Unit tests are disabled. Configure without '--disable-unit-tests', or avoid calling 'make check'."
+	@exit 1
+endif
+
+ifneq ($(ENABLE_FUNCTIONAL_TESTS), yes)
+.PHONY: installcheck
+installcheck:
+	@echo "Functional tests are disabled. Configure without '--disable-functional-tests', or avoid calling 'make installcheck'."
+	@exit 1
+endif
+
 # Must be included after `mk/lib.mk` so rules refer to variables defined
 # by the library. Rules are not "lazy" like variables, unfortunately.
-ifeq ($(ENABLE_DOC_GEN),yes)
+
+ifeq ($(ENABLE_DOC_GEN), yes)
 $(eval $(call include-sub-makefile, doc/manual/local.mk))
 else
 .PHONY: manual-html manpages
@@ -75,7 +81,7 @@ manual-html manpages:
 	@exit 1
 endif
 
-ifeq ($(ENABLE_INTERNAL_API_DOCS),yes)
+ifeq ($(ENABLE_INTERNAL_API_DOCS), yes)
 $(eval $(call include-sub-makefile, doc/internal-api/local.mk))
 else
 .PHONY: internal-api-html

From ea454d8687b96376b221d7bdb1085968867c2496 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <contact@infinisil.com>
Date: Wed, 20 Dec 2023 03:24:38 +0100
Subject: [PATCH 273/421] Undeprecate isNull

There's no good reason to deprecate it:
- For consistency reasons it should continue to exist, such that all
primitive types have a corresponding `builtins.is*` primop.
- There's no implementation cost to continuing to have this function
- It costs users time to try to migrate away from it, e.g.
  https://github.com/NixOS/nixpkgs/pull/219747 and https://github.com/NixOS/nixpkgs/pull/275548
- Using it can give easier-to-read code like `all isNull list`

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 src/libexpr/primops.cc | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 8b689f0c8..1ca4a2541 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -438,9 +438,7 @@ static RegisterPrimOp primop_isNull({
     .doc = R"(
       Return `true` if *e* evaluates to `null`, and `false` otherwise.
 
-      > **Warning**
-      >
-      > This function is *deprecated*; just write `e == null` instead.
+      This is equivalent to `e == null`.
     )",
     .fun = prim_isNull,
 });

From d77a39a314871b9c9a0a4d09b153c40ea9c8aaca Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 21 Dec 2023 16:22:34 +0100
Subject: [PATCH 274/421] Fix indent

---
 src/libutil/url-name.cc | 45 +++++++++++++++++++++--------------------
 src/libutil/url-name.hh |  2 +-
 2 files changed, 24 insertions(+), 23 deletions(-)

diff --git a/src/libutil/url-name.cc b/src/libutil/url-name.cc
index f94383e32..7e51aa2e1 100644
--- a/src/libutil/url-name.cc
+++ b/src/libutil/url-name.cc
@@ -13,35 +13,36 @@ static const std::regex gitProviderRegex("github|gitlab|sourcehut");
 static const std::regex gitSchemeRegex("git($|\\+.*)");
 static const std::regex defaultOutputRegex(".*\\.default($|\\^.*)");
 
-std::optional<std::string> getNameFromURL(ParsedURL url) {
-  std::smatch match;
+std::optional<std::string> getNameFromURL(const ParsedURL & url)
+{
+    std::smatch match;
 
-  /* If there is a dir= argument, use its value */
-  if (url.query.count("dir") > 0)
-    return url.query.at("dir");
+    /* If there is a dir= argument, use its value */
+    if (url.query.count("dir") > 0)
+        return url.query.at("dir");
 
-  /* If the fragment isn't a "default" and contains two attribute elements, use the last one */
-  if (std::regex_match(url.fragment, match, lastAttributeRegex))
-    return match.str(1);
+    /* If the fragment isn't a "default" and contains two attribute elements, use the last one */
+    if (std::regex_match(url.fragment, match, lastAttributeRegex))
+        return match.str(1);
 
-  /* If this is a github/gitlab/sourcehut flake, use the repo name */
-  if (std::regex_match(url.scheme, gitProviderRegex) && std::regex_match(url.path, match, secondPathSegmentRegex))
-    return match.str(1);
+    /* If this is a github/gitlab/sourcehut flake, use the repo name */
+    if (std::regex_match(url.scheme, gitProviderRegex) && std::regex_match(url.path, match, secondPathSegmentRegex))
+        return match.str(1);
 
-  /* If it is a regular git flake, use the directory name */
-  if (std::regex_match(url.scheme, gitSchemeRegex) && std::regex_match(url.path, match, lastPathSegmentRegex))
-    return match.str(1);
+    /* If it is a regular git flake, use the directory name */
+    if (std::regex_match(url.scheme, gitSchemeRegex) && std::regex_match(url.path, match, lastPathSegmentRegex))
+        return match.str(1);
 
-  /* If everything failed but there is a non-default fragment, use it in full */
-  if (!url.fragment.empty() && !std::regex_match(url.fragment, defaultOutputRegex))
-    return url.fragment;
+    /* If everything failed but there is a non-default fragment, use it in full */
+    if (!url.fragment.empty() && !std::regex_match(url.fragment, defaultOutputRegex))
+        return url.fragment;
 
-  /* If there is no fragment, take the last element of the path */
-  if (std::regex_match(url.path, match, lastPathSegmentRegex))
-    return match.str(1);
+    /* If there is no fragment, take the last element of the path */
+    if (std::regex_match(url.path, match, lastPathSegmentRegex))
+        return match.str(1);
 
-  /* If even that didn't work, the URL does not contain enough info to determine a useful name */
-  return {};
+    /* If even that didn't work, the URL does not contain enough info to determine a useful name */
+    return {};
 }
 
 }
diff --git a/src/libutil/url-name.hh b/src/libutil/url-name.hh
index 188b951e5..6f32754d2 100644
--- a/src/libutil/url-name.hh
+++ b/src/libutil/url-name.hh
@@ -15,6 +15,6 @@ namespace nix {
  * flake output, for example because it is empty or "default".
  * Otherwise returns the extracted name.
  */
-std::optional<std::string> getNameFromURL(ParsedURL url);
+std::optional<std::string> getNameFromURL(const ParsedURL & url);
 
 }

From 14508ade289b884ab2ceb0645a549a69fba82cab Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 21 Dec 2023 16:25:55 +0100
Subject: [PATCH 275/421] Typo

---
 src/nix/profile.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 98fa165e8..1d89815e2 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -119,7 +119,7 @@ struct ProfileManifest
 
         if (pathExists(manifestPath)) {
             auto json = nlohmann::json::parse(readFile(manifestPath));
-            /* Keep track of alreay found names to allow preventing duplicates */
+            /* Keep track of already found names to allow preventing duplicates. */
             std::set<std::string> foundNames;
 
             auto version = json.value("version", 0);

From 942d635102810a310c747cee66d9e9f343e6b4c3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 21 Dec 2023 16:33:53 +0100
Subject: [PATCH 276/421] Fix release notes

---
 doc/manual/rl-next/nix-profile-names.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/rl-next/nix-profile-names.md b/doc/manual/rl-next/nix-profile-names.md
index 2f4a589a7..f5953bd72 100644
--- a/doc/manual/rl-next/nix-profile-names.md
+++ b/doc/manual/rl-next/nix-profile-names.md
@@ -1,5 +1,5 @@
 ---
-synopsis: nix profile: Allow referring to elements by human-readable name
+synopsis: "`nix profile` now allows referring to elements by human-readable name"
 prs: 8678
 ---
 

From 5ed1884875cc6a6e9330b6c5a2f24c35e685f5a0 Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.e.helbling@outlook.com>
Date: Thu, 21 Dec 2023 10:14:54 -0800
Subject: [PATCH 277/421] libcmd: Installable::toStorePaths ->
 Installable::toStorePathSet

---
 src/libcmd/installables.cc | 4 ++--
 src/libcmd/installables.hh | 2 +-
 src/nix/develop.cc         | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 6b3c82374..be9ebe9ca 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -715,7 +715,7 @@ BuiltPaths Installable::toBuiltPaths(
     }
 }
 
-StorePathSet Installable::toStorePaths(
+StorePathSet Installable::toStorePathSet(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode, OperateOn operateOn,
@@ -735,7 +735,7 @@ StorePath Installable::toStorePath(
     Realise mode, OperateOn operateOn,
     ref<Installable> installable)
 {
-    auto paths = toStorePaths(evalStore, store, mode, operateOn, {installable});
+    auto paths = toStorePathSet(evalStore, store, mode, operateOn, {installable});
 
     if (paths.size() != 1)
         throw Error("argument '%s' should evaluate to one store path", installable->what());
diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh
index e087f935c..c8ad41388 100644
--- a/src/libcmd/installables.hh
+++ b/src/libcmd/installables.hh
@@ -165,7 +165,7 @@ struct Installable
         const Installables & installables,
         BuildMode bMode = bmNormal);
 
-    static std::set<StorePath> toStorePaths(
+    static std::set<StorePath> toStorePathSet(
         ref<Store> evalStore,
         ref<Store> store,
         Realise mode,
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 8db2de491..974020951 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -376,7 +376,7 @@ struct Common : InstallableCommand, MixProfile
         for (auto & [installable_, dir_] : redirects) {
             auto dir = absPath(dir_);
             auto installable = parseInstallable(store, installable_);
-            auto builtPaths = Installable::toStorePaths(
+            auto builtPaths = Installable::toStorePathSet(
                 getEvalStore(), store, Realise::Nothing, OperateOn::Output, {installable});
             for (auto & path: builtPaths) {
                 auto from = store->printStorePath(path);
@@ -631,7 +631,7 @@ struct CmdDevelop : Common, MixEnvironment
 
             bool found = false;
 
-            for (auto & path : Installable::toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, {bashInstallable})) {
+            for (auto & path : Installable::toStorePathSet(getEvalStore(), store, Realise::Outputs, OperateOn::Output, {bashInstallable})) {
                 auto s = store->printStorePath(path) + "/bin/bash";
                 if (pathExists(s)) {
                     shell = s;

From 1fb43d1eee6f398686523c0bb80adb987c584c61 Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.e.helbling@outlook.com>
Date: Wed, 20 Dec 2023 10:25:22 -0800
Subject: [PATCH 278/421] tests: add a test for command line ordering

---
 tests/functional/shell-hello.nix | 16 ++++++++++++++++
 tests/functional/shell.sh        |  8 ++++++++
 2 files changed, 24 insertions(+)

diff --git a/tests/functional/shell-hello.nix b/tests/functional/shell-hello.nix
index 3fdd3501d..dfe66ef93 100644
--- a/tests/functional/shell-hello.nix
+++ b/tests/functional/shell-hello.nix
@@ -23,4 +23,20 @@ with import ./config.nix;
         chmod +x $dev/bin/hello2
       '';
   };
+
+  salve-mundi = mkDerivation {
+    name = "salve-mundi";
+    outputs = [ "out" ];
+    meta.outputsToInstall = [ "out" ];
+    buildCommand =
+      ''
+        mkdir -p $out/bin
+
+        cat > $out/bin/hello <<EOF
+        #! ${shell}
+        echo "Salve Mundi from $out/bin/hello"
+        EOF
+        chmod +x $out/bin/hello
+      '';
+  };
 }
diff --git a/tests/functional/shell.sh b/tests/functional/shell.sh
index d2f7cf14e..8bbeabedf 100644
--- a/tests/functional/shell.sh
+++ b/tests/functional/shell.sh
@@ -10,6 +10,14 @@ nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS'
 nix shell -f shell-hello.nix hello^dev -c hello2 | grep 'Hello2'
 nix shell -f shell-hello.nix 'hello^*' -c hello2 | grep 'Hello2'
 
+
+if isDaemonNewer "2.20.0pre20231220"; then
+    # Test that command line attribute ordering is reflected in the PATH
+    # https://github.com/NixOS/nix/issues/7905
+    nix shell -f shell-hello.nix hello salve-mundi -c hello | grep 'Hello World'
+    nix shell -f shell-hello.nix salve-mundi hello -c hello | grep 'Salve Mundi'
+fi
+
 requireSandboxSupport
 
 chmod -R u+w $TEST_ROOT/store0 || true

From f4454aac9f002c33b84bb52e0bba2a46ea6170cd Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.e.helbling@outlook.com>
Date: Mon, 18 Dec 2023 15:22:09 -0800
Subject: [PATCH 279/421] nix shell: reflect command line order in PATH order

Prior to this change, Nix would prepend every installable to the PATH
list in order to ensure that installables appeared before the current
PATH from the ambient environment.

With this change, all the installables are still prepended to the PATH,
but in the same order as they appear on the command line. This means
that the first of two packages that expose an executable `hello` would
appear in the PATH first, and thus be executed first.

See the test in the prior commit for a more concrete example.
---
 src/libcmd/installables.cc | 14 ++++++++++++++
 src/libcmd/installables.hh |  7 +++++++
 src/nix/run.cc             |  9 ++++++---
 3 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index be9ebe9ca..736c41a1e 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -729,6 +729,20 @@ StorePathSet Installable::toStorePathSet(
     return outPaths;
 }
 
+StorePaths Installable::toStorePaths(
+    ref<Store> evalStore,
+    ref<Store> store,
+    Realise mode, OperateOn operateOn,
+    const Installables & installables)
+{
+    StorePaths outPaths;
+    for (auto & path : toBuiltPaths(evalStore, store, mode, operateOn, installables)) {
+        auto thisOutPaths = path.outPaths();
+        outPaths.insert(outPaths.end(), thisOutPaths.begin(), thisOutPaths.end());
+    }
+    return outPaths;
+}
+
 StorePath Installable::toStorePath(
     ref<Store> evalStore,
     ref<Store> store,
diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh
index c8ad41388..95e8841ca 100644
--- a/src/libcmd/installables.hh
+++ b/src/libcmd/installables.hh
@@ -172,6 +172,13 @@ struct Installable
         OperateOn operateOn,
         const Installables & installables);
 
+    static std::vector<StorePath> toStorePaths(
+        ref<Store> evalStore,
+        ref<Store> store,
+        Realise mode,
+        OperateOn operateOn,
+        const Installables & installables);
+
     static StorePath toStorePath(
         ref<Store> evalStore,
         ref<Store> store,
diff --git a/src/nix/run.cc b/src/nix/run.cc
index efc0c56a1..9bca5b9d0 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -114,7 +114,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment
 
         setEnviron();
 
-        auto unixPath = tokenizeString<Strings>(getEnv("PATH").value_or(""), ":");
+        std::vector<std::string> pathAdditions;
 
         while (!todo.empty()) {
             auto path = todo.front();
@@ -122,7 +122,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment
             if (!done.insert(path).second) continue;
 
             if (true)
-                unixPath.push_front(store->printStorePath(path) + "/bin");
+                pathAdditions.push_back(store->printStorePath(path) + "/bin");
 
             auto propPath = CanonPath(store->printStorePath(path)) + "nix-support" + "propagated-user-env-packages";
             if (auto st = accessor->maybeLstat(propPath); st && st->type == SourceAccessor::tRegular) {
@@ -131,7 +131,10 @@ struct CmdShell : InstallablesCommand, MixEnvironment
             }
         }
 
-        setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1);
+        auto unixPath = tokenizeString<Strings>(getEnv("PATH").value_or(""), ":");
+        unixPath.insert(unixPath.begin(), pathAdditions.begin(), pathAdditions.end());
+        auto unixPathString = concatStringsSep(":", unixPath);
+        setenv("PATH", unixPathString.c_str(), 1);
 
         Strings args;
         for (auto & arg : command) args.push_back(arg);

From 8c4ea12f11511519726737cc39bc5b4e089b9f33 Mon Sep 17 00:00:00 2001
From: Robert Hensing <roberth@users.noreply.github.com>
Date: Thu, 21 Dec 2023 21:03:06 +0100
Subject: [PATCH 280/421] libutil/url-parts.hh: comment

---
 src/libutil/url-parts.hh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh
index e968eea4b..a3b4f5b99 100644
--- a/src/libutil/url-parts.hh
+++ b/src/libutil/url-parts.hh
@@ -25,6 +25,7 @@ const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRege
 
 /// A Git ref (i.e. branch or tag name).
 /// \todo check that this is correct.
+/// This regex incomplete. See https://git-scm.com/docs/git-check-ref-format
 const static std::string refRegexS = "[a-zA-Z0-9@][a-zA-Z0-9_.\\/@-+]*";
 extern std::regex refRegex;
 

From 4f47152209a81a2bef421467ca4bec00023eec04 Mon Sep 17 00:00:00 2001
From: Robert Hensing <roberth@users.noreply.github.com>
Date: Thu, 21 Dec 2023 23:11:25 +0100
Subject: [PATCH 281/421] libutil/url-parts.hh: Fix regex

Regex syntax is awful.
---
 src/libutil/url-parts.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh
index a3b4f5b99..4bb37ea9b 100644
--- a/src/libutil/url-parts.hh
+++ b/src/libutil/url-parts.hh
@@ -26,7 +26,7 @@ const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRege
 /// A Git ref (i.e. branch or tag name).
 /// \todo check that this is correct.
 /// This regex incomplete. See https://git-scm.com/docs/git-check-ref-format
-const static std::string refRegexS = "[a-zA-Z0-9@][a-zA-Z0-9_.\\/@-+]*";
+const static std::string refRegexS = "[a-zA-Z0-9@][a-zA-Z0-9_.\\/@+-]*";
 extern std::regex refRegex;
 
 /// Instead of defining what a good Git Ref is, we define what a bad Git Ref is

From 26d7b0c793b389b71218fc38e613d3f75ad72299 Mon Sep 17 00:00:00 2001
From: Felix Uhl <github@mail.felix-uhl.de>
Date: Thu, 21 Dec 2023 22:45:21 +0100
Subject: [PATCH 282/421] Move url-name utility to libexpr/flake

---
 src/{libutil => libexpr/flake}/url-name.cc        | 0
 src/{libutil => libexpr/flake}/url-name.hh        | 0
 src/nix/profile.cc                                | 2 +-
 tests/unit/{libutil => libexpr/flake}/url-name.cc | 2 +-
 tests/unit/libexpr/local.mk                       | 3 ++-
 5 files changed, 4 insertions(+), 3 deletions(-)
 rename src/{libutil => libexpr/flake}/url-name.cc (100%)
 rename src/{libutil => libexpr/flake}/url-name.hh (100%)
 rename tests/unit/{libutil => libexpr/flake}/url-name.cc (99%)

diff --git a/src/libutil/url-name.cc b/src/libexpr/flake/url-name.cc
similarity index 100%
rename from src/libutil/url-name.cc
rename to src/libexpr/flake/url-name.cc
diff --git a/src/libutil/url-name.hh b/src/libexpr/flake/url-name.hh
similarity index 100%
rename from src/libutil/url-name.hh
rename to src/libexpr/flake/url-name.hh
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 1d89815e2..abd56e4f4 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -11,7 +11,7 @@
 #include "profiles.hh"
 #include "names.hh"
 #include "url.hh"
-#include "url-name.hh"
+#include "flake/url-name.hh"
 
 #include <nlohmann/json.hpp>
 #include <regex>
diff --git a/tests/unit/libutil/url-name.cc b/tests/unit/libexpr/flake/url-name.cc
similarity index 99%
rename from tests/unit/libutil/url-name.cc
rename to tests/unit/libexpr/flake/url-name.cc
index f637efa89..84d32837c 100644
--- a/tests/unit/libutil/url-name.cc
+++ b/tests/unit/libexpr/flake/url-name.cc
@@ -1,4 +1,4 @@
-#include "url-name.hh"
+#include "flake/url-name.hh"
 #include <gtest/gtest.h>
 
 namespace nix {
diff --git a/tests/unit/libexpr/local.mk b/tests/unit/libexpr/local.mk
index 5743880d7..25810ad9c 100644
--- a/tests/unit/libexpr/local.mk
+++ b/tests/unit/libexpr/local.mk
@@ -16,7 +16,8 @@ endif
 
 libexpr-tests_SOURCES := \
     $(wildcard $(d)/*.cc) \
-    $(wildcard $(d)/value/*.cc)
+    $(wildcard $(d)/value/*.cc) \
+    $(wildcard $(d)/flake/*.cc)
 
 libexpr-tests_EXTRA_INCLUDES = \
     -I tests/unit/libexpr-support \

From 4b4111866358b39d5bfc352fa58040c8a54a2759 Mon Sep 17 00:00:00 2001
From: Felix Uhl <github@mail.felix-uhl.de>
Date: Fri, 22 Dec 2023 09:38:13 +0100
Subject: [PATCH 283/421] Move flakeref tests to new flake/ subdirectory

---
 tests/unit/libexpr/{ => flake}/flakeref.cc | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename tests/unit/libexpr/{ => flake}/flakeref.cc (100%)

diff --git a/tests/unit/libexpr/flakeref.cc b/tests/unit/libexpr/flake/flakeref.cc
similarity index 100%
rename from tests/unit/libexpr/flakeref.cc
rename to tests/unit/libexpr/flake/flakeref.cc

From 3187bc9ac3dd193b9329ef68c73ac3cca794ed78 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 21 Dec 2023 16:48:29 +0100
Subject: [PATCH 284/421] nix profile: Remove indices

---
 src/nix/profile-list.md         |  2 --
 src/nix/profile-remove.md       |  7 ------
 src/nix/profile-upgrade.md      |  7 ------
 src/nix/profile.cc              | 39 ++++++++++-----------------------
 tests/functional/nix-profile.sh | 11 +++++-----
 5 files changed, 16 insertions(+), 50 deletions(-)

diff --git a/src/nix/profile-list.md b/src/nix/profile-list.md
index facfdf0d6..9811b9ec9 100644
--- a/src/nix/profile-list.md
+++ b/src/nix/profile-list.md
@@ -7,14 +7,12 @@ R""(
   ```console
   # nix profile list
   Name:               gdb
-  Index:              0
   Flake attribute:    legacyPackages.x86_64-linux.gdb
   Original flake URL: flake:nixpkgs
   Locked flake URL:   github:NixOS/nixpkgs/7b38b03d76ab71bdc8dc325e3f6338d984cc35ca
   Store paths:        /nix/store/indzcw5wvlhx6vwk7k4iq29q15chvr3d-gdb-11.1
 
   Name:               blender-bin
-  Index:              1
   Flake attribute:    packages.x86_64-linux.default
   Original flake URL: flake:blender-bin
   Locked flake URL:   github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender
diff --git a/src/nix/profile-remove.md b/src/nix/profile-remove.md
index c994b79bd..1f6532250 100644
--- a/src/nix/profile-remove.md
+++ b/src/nix/profile-remove.md
@@ -8,13 +8,6 @@ R""(
   # nix profile remove hello
   ```
 
-* Remove a package by index
-  *(deprecated, will be removed in a future version)*:
-
-  ```console
-  # nix profile remove 3
-  ```
-
 * Remove all packages:
 
   ```console
diff --git a/src/nix/profile-upgrade.md b/src/nix/profile-upgrade.md
index 47103edfc..432b8fa94 100644
--- a/src/nix/profile-upgrade.md
+++ b/src/nix/profile-upgrade.md
@@ -15,13 +15,6 @@ R""(
   # nix profile upgrade hello
   ```
 
-* Upgrade a specific package by index
-  *(deprecated, will be removed in a future version)*:
-
-  ```console
-  # nix profile upgrade 0
-  ```
-
 # Description
 
 This command upgrades a previously installed package in a Nix profile,
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 1d89815e2..517693cd4 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -470,40 +470,28 @@ public:
         std::string pattern;
         std::regex  reg;
     };
-    typedef std::variant<size_t, Path, RegexPattern> Matcher;
+    typedef std::variant<Path, RegexPattern> Matcher;
 
     std::vector<Matcher> getMatchers(ref<Store> store)
     {
         std::vector<Matcher> res;
 
-        auto anyIndexMatchers = false;
-
         for (auto & s : _matchers) {
-            if (auto n = string2Int<size_t>(s)) {
-                res.push_back(*n);
-                anyIndexMatchers = true;
-            }
+            if (auto n = string2Int<size_t>(s))
+                throw Error("'nix profile' no longer supports indices ('%d')", *n);
             else if (store->isStorePath(s))
                 res.push_back(s);
             else
                 res.push_back(RegexPattern{s,std::regex(s, std::regex::extended | std::regex::icase)});
         }
 
-        if (anyIndexMatchers) {
-            warn("Indices are deprecated and will be removed in a future version!\n"
-                 "         Refer to packages by their `Name` as printed by `nix profile list`.\n"
-                 "         See https://github.com/NixOS/nix/issues/9171 for more information.");
-        }
-
         return res;
     }
 
-    bool matches(const Store & store, const ProfileElement & element, size_t pos, const std::vector<Matcher> & matchers)
+    bool matches(const Store & store, const ProfileElement & element, const std::vector<Matcher> & matchers)
     {
         for (auto & matcher : matchers) {
-            if (auto n = std::get_if<size_t>(&matcher)) {
-                if (*n == pos) return true;
-            } else if (auto path = std::get_if<Path>(&matcher)) {
+            if (auto path = std::get_if<Path>(&matcher)) {
                 if (element.storePaths.count(store.parseStorePath(*path))) return true;
             } else if (auto regex = std::get_if<RegexPattern>(&matcher)) {
                 if (std::regex_match(element.name, regex->reg))
@@ -539,7 +527,7 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem
 
         for (size_t i = 0; i < oldManifest.elements.size(); ++i) {
             auto & element(oldManifest.elements[i]);
-            if (!matches(*store, element, i, matchers)) {
+            if (!matches(*store, element, matchers)) {
                 newManifest.elements.push_back(std::move(element));
             } else {
                 notice("removing '%s'", element.identifier());
@@ -553,11 +541,9 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem
 
         if (removedCount == 0) {
             for (auto matcher: matchers) {
-                if (const size_t * index = std::get_if<size_t>(&matcher)){
-                    warn("'%d' is not a valid index", *index);
-                } else if (const Path * path = std::get_if<Path>(&matcher)){
+                if (const Path * path = std::get_if<Path>(&matcher)) {
                     warn("'%s' does not match any paths", *path);
-                } else if (const RegexPattern * regex = std::get_if<RegexPattern>(&matcher)){
+                } else if (const RegexPattern * regex = std::get_if<RegexPattern>(&matcher)) {
                     warn("'%s' does not match any packages", regex->pattern);
                 }
             }
@@ -595,7 +581,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
 
         for (size_t i = 0; i < manifest.elements.size(); ++i) {
             auto & element(manifest.elements[i]);
-            if (!matches(*store, element, i, matchers)) {
+            if (!matches(*store, element, matchers)) {
                 continue;
             }
 
@@ -657,11 +643,9 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
         if (upgradedCount == 0) {
             if (matchedCount == 0) {
                 for (auto & matcher : matchers) {
-                    if (const size_t * index = std::get_if<size_t>(&matcher)){
-                        warn("'%d' is not a valid index", *index);
-                    } else if (const Path * path = std::get_if<Path>(&matcher)){
+                    if (const Path * path = std::get_if<Path>(&matcher)) {
                         warn("'%s' does not match any paths", *path);
-                    } else if (const RegexPattern * regex = std::get_if<RegexPattern>(&matcher)){
+                    } else if (const RegexPattern * regex = std::get_if<RegexPattern>(&matcher)) {
                         warn("'%s' does not match any packages", regex->pattern);
                     }
                 }
@@ -715,7 +699,6 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro
                 logger->cout("Name:               " ANSI_BOLD "%s" ANSI_NORMAL "%s",
                     element.name,
                     element.active ? "" : " " ANSI_RED "(inactive)" ANSI_NORMAL);
-                logger->cout("Index:              %s", i);
                 if (element.source) {
                     logger->cout("Flake attribute:    %s%s", element.source->attrPath, element.source->outputs.to_string());
                     logger->cout("Original flake URL: %s", element.source->originalRef.to_string());
diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh
index eced4d3f1..618b6241d 100644
--- a/tests/functional/nix-profile.sh
+++ b/tests/functional/nix-profile.sh
@@ -49,7 +49,7 @@ cp ./config.nix $flake1Dir/
 nix-env -f ./user-envs.nix -i foo-1.0
 nix profile list | grep -A2 'Name:.*foo' | grep 'Store paths:.*foo-1.0'
 nix profile install $flake1Dir -L
-nix profile list | grep -A4 'Index:.*1' | grep 'Locked flake URL:.*narHash'
+nix profile list | grep -A4 'Name:.*flake1' | grep 'Locked flake URL:.*narHash'
 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]]
 [ -e $TEST_HOME/.nix-profile/share/man ]
 (! [ -e $TEST_HOME/.nix-profile/include ])
@@ -58,9 +58,8 @@ nix profile history | grep "packages.$system.default: ∅ -> 1.0"
 nix profile diff-closures | grep 'env-manifest.nix: ε → ∅'
 
 # Test XDG Base Directories support
-
 export NIX_CONFIG="use-xdg-base-directories = true"
-nix profile remove 1
+nix profile remove flake1
 nix profile install $flake1Dir
 [[ $($TEST_HOME/.local/state/nix/profile/bin/hello) = "Hello World" ]]
 unset NIX_CONFIG
@@ -68,7 +67,7 @@ unset NIX_CONFIG
 # Test upgrading a package.
 printf NixOS > $flake1Dir/who
 printf 2.0 > $flake1Dir/version
-nix profile upgrade 1
+nix profile upgrade flake1
 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello NixOS" ]]
 nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 2.0, 2.0-man"
 
@@ -89,7 +88,7 @@ nix profile diff-closures | grep 'Version 3 -> 4'
 # Test installing a non-flake package.
 nix profile install --file ./simple.nix ''
 [[ $(cat $TEST_HOME/.nix-profile/hello) = "Hello World!" ]]
-nix profile remove 1
+nix profile remove simple
 nix profile install $(nix-build --no-out-link ./simple.nix)
 [[ $(cat $TEST_HOME/.nix-profile/hello) = "Hello World!" ]]
 
@@ -107,7 +106,7 @@ nix profile wipe-history
 # Test upgrade to CA package.
 printf true > $flake1Dir/ca.nix
 printf 3.0 > $flake1Dir/version
-nix profile upgrade 0
+nix profile upgrade flake1
 nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 3.0, 3.0-man"
 
 # Test new install of CA package.

From 6268a45b650f563bae2360e0540920a2959bdd40 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 22 Dec 2023 16:11:25 +0100
Subject: [PATCH 285/421] nix profile: Make profile element names stable

The profile manifest is now an object keyed on the name returned by
getNameFromURL() at installation time, instead of an array. This
ensures that the names of profile elements don't change when other
elements are added/removed.
---
 src/nix/profile.cc              | 140 ++++++++++++++++----------------
 tests/functional/nix-profile.sh |  17 ++--
 2 files changed, 80 insertions(+), 77 deletions(-)

diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 517693cd4..8b3918b80 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -45,7 +45,6 @@ const int defaultPriority = 5;
 struct ProfileElement
 {
     StorePathSet storePaths;
-    std::string name;
     std::optional<ProfileElementSource> source;
     bool active = true;
     int priority = defaultPriority;
@@ -82,11 +81,6 @@ struct ProfileElement
         return showVersions(versions);
     }
 
-    bool operator < (const ProfileElement & other) const
-    {
-        return std::tuple(identifier(), storePaths) < std::tuple(other.identifier(), other.storePaths);
-    }
-
     void updateStorePaths(
         ref<Store> evalStore,
         ref<Store> store,
@@ -109,7 +103,9 @@ struct ProfileElement
 
 struct ProfileManifest
 {
-    std::vector<ProfileElement> elements;
+    using ProfileElementName = std::string;
+
+    std::map<ProfileElementName, ProfileElement> elements;
 
     ProfileManifest() { }
 
@@ -119,8 +115,6 @@ struct ProfileManifest
 
         if (pathExists(manifestPath)) {
             auto json = nlohmann::json::parse(readFile(manifestPath));
-            /* Keep track of already found names to allow preventing duplicates. */
-            std::set<std::string> foundNames;
 
             auto version = json.value("version", 0);
             std::string sUrl;
@@ -131,6 +125,7 @@ struct ProfileManifest
                     sOriginalUrl = "originalUri";
                     break;
                 case 2:
+                case 3:
                     sUrl = "url";
                     sOriginalUrl = "originalUrl";
                     break;
@@ -138,7 +133,9 @@ struct ProfileManifest
                     throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version);
             }
 
-            for (auto & e : json["elements"]) {
+            auto elems = json["elements"];
+            for (auto & elem : elems.items()) {
+                auto & e = elem.value();
                 ProfileElement element;
                 for (auto & p : e["storePaths"])
                     element.storePaths.insert(state.store->parseStorePath((std::string) p));
@@ -155,25 +152,16 @@ struct ProfileManifest
                     };
                 }
 
-                std::string nameCandidate = element.identifier();
-                if (e.contains("name")) {
-                    nameCandidate = e["name"];
-                }
-                else if (element.source) {
-                    auto url = parseURL(element.source->to_string());
-                    auto name = getNameFromURL(url);
-                    if (name)
-                        nameCandidate = *name;
-                }
+                std::string name =
+                    elems.is_object()
+                    ? elem.key()
+                    : e.contains("name")
+                    ? (std::string) e["name"]
+                    : element.source
+                    ? getNameFromURL(parseURL(element.source->to_string())).value_or(element.identifier())
+                    : element.identifier();
 
-                auto finalName = nameCandidate;
-                for (int i = 1; foundNames.contains(finalName); ++i) {
-                    finalName = nameCandidate + std::to_string(i);
-                }
-                element.name = finalName;
-                foundNames.insert(element.name);
-
-                elements.emplace_back(std::move(element));
+                addElement(name, std::move(element));
             }
         }
 
@@ -187,16 +175,34 @@ struct ProfileManifest
             for (auto & drvInfo : drvInfos) {
                 ProfileElement element;
                 element.storePaths = {drvInfo.queryOutPath()};
-                element.name = element.identifier();
-                elements.emplace_back(std::move(element));
+                addElement(std::move(element));
             }
         }
     }
 
+    void addElement(std::string_view nameCandidate, ProfileElement element)
+    {
+        std::string finalName(nameCandidate);
+        for (int i = 1; elements.contains(finalName); ++i)
+            finalName = nameCandidate + "-" + std::to_string(i);
+
+        elements.insert_or_assign(finalName, std::move(element));
+    }
+
+    void addElement(ProfileElement element)
+    {
+        auto name =
+            element.source
+            ? getNameFromURL(parseURL(element.source->to_string()))
+            : std::nullopt;
+        auto name2 = name ? *name : element.identifier();
+        addElement(name2, std::move(element));
+    }
+
     nlohmann::json toJSON(Store & store) const
     {
-        auto array = nlohmann::json::array();
-        for (auto & element : elements) {
+        auto es = nlohmann::json::object();
+        for (auto & [name, element] : elements) {
             auto paths = nlohmann::json::array();
             for (auto & path : element.storePaths)
                 paths.push_back(store.printStorePath(path));
@@ -210,11 +216,11 @@ struct ProfileManifest
                 obj["attrPath"] = element.source->attrPath;
                 obj["outputs"] = element.source->outputs;
             }
-            array.push_back(obj);
+            es[name] = obj;
         }
         nlohmann::json json;
-        json["version"] = 2;
-        json["elements"] = array;
+        json["version"] = 3;
+        json["elements"] = es;
         return json;
     }
 
@@ -225,7 +231,7 @@ struct ProfileManifest
         StorePathSet references;
 
         Packages pkgs;
-        for (auto & element : elements) {
+        for (auto & [name, element] : elements) {
             for (auto & path : element.storePaths) {
                 if (element.active)
                     pkgs.emplace_back(store->printStorePath(path), true, element.priority);
@@ -267,33 +273,27 @@ struct ProfileManifest
 
     static void printDiff(const ProfileManifest & prev, const ProfileManifest & cur, std::string_view indent)
     {
-        auto prevElems = prev.elements;
-        std::sort(prevElems.begin(), prevElems.end());
-
-        auto curElems = cur.elements;
-        std::sort(curElems.begin(), curElems.end());
-
-        auto i = prevElems.begin();
-        auto j = curElems.begin();
+        auto i = prev.elements.begin();
+        auto j = cur.elements.begin();
 
         bool changes = false;
 
-        while (i != prevElems.end() || j != curElems.end()) {
-            if (j != curElems.end() && (i == prevElems.end() || i->identifier() > j->identifier())) {
-                logger->cout("%s%s: ∅ -> %s", indent, j->identifier(), j->versions());
+        while (i != prev.elements.end() || j != cur.elements.end()) {
+            if (j != cur.elements.end() && (i == prev.elements.end() || i->first > j->first)) {
+                logger->cout("%s%s: ∅ -> %s", indent, j->second.identifier(), j->second.versions());
                 changes = true;
                 ++j;
             }
-            else if (i != prevElems.end() && (j == curElems.end() || i->identifier() < j->identifier())) {
-                logger->cout("%s%s: %s -> ∅", indent, i->identifier(), i->versions());
+            else if (i != prev.elements.end() && (j == cur.elements.end() || i->first < j->first)) {
+                logger->cout("%s%s: %s -> ∅", indent, i->second.identifier(), i->second.versions());
                 changes = true;
                 ++i;
             }
             else {
-                auto v1 = i->versions();
-                auto v2 = j->versions();
+                auto v1 = i->second.versions();
+                auto v2 = j->second.versions();
                 if (v1 != v2) {
-                    logger->cout("%s%s: %s -> %s", indent, i->identifier(), v1, v2);
+                    logger->cout("%s%s: %s -> %s", indent, i->second.identifier(), v1, v2);
                     changes = true;
                 }
                 ++i;
@@ -392,7 +392,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
 
             element.updateStorePaths(getEvalStore(), store, res);
 
-            manifest.elements.push_back(std::move(element));
+            manifest.addElement(std::move(element));
         }
 
         try {
@@ -402,7 +402,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
             //       See https://github.com/NixOS/nix/compare/3efa476c5439f8f6c1968a6ba20a31d1239c2f04..1fe5d172ece51a619e879c4b86f603d9495cc102
             auto findRefByFilePath = [&]<typename Iterator>(Iterator begin, Iterator end) {
                 for (auto it = begin; it != end; it++) {
-                    auto profileElement = *it;
+                    auto & profileElement = it->second;
                     for (auto & storePath : profileElement.storePaths) {
                         if (conflictError.fileA.starts_with(store->printStorePath(storePath))) {
                             return std::pair(conflictError.fileA, profileElement.toInstallables(*store));
@@ -488,13 +488,17 @@ public:
         return res;
     }
 
-    bool matches(const Store & store, const ProfileElement & element, const std::vector<Matcher> & matchers)
+    bool matches(
+        const Store & store,
+        const std::string & name,
+        const ProfileElement & element,
+        const std::vector<Matcher> & matchers)
     {
         for (auto & matcher : matchers) {
             if (auto path = std::get_if<Path>(&matcher)) {
                 if (element.storePaths.count(store.parseStorePath(*path))) return true;
             } else if (auto regex = std::get_if<RegexPattern>(&matcher)) {
-                if (std::regex_match(element.name, regex->reg))
+                if (std::regex_match(name, regex->reg))
                     return true;
             }
         }
@@ -525,10 +529,9 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem
 
         ProfileManifest newManifest;
 
-        for (size_t i = 0; i < oldManifest.elements.size(); ++i) {
-            auto & element(oldManifest.elements[i]);
-            if (!matches(*store, element, matchers)) {
-                newManifest.elements.push_back(std::move(element));
+        for (auto & [name, element] : oldManifest.elements) {
+            if (!matches(*store, name, element, matchers)) {
+                newManifest.elements.insert_or_assign(name, std::move(element));
             } else {
                 notice("removing '%s'", element.identifier());
             }
@@ -574,14 +577,13 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
         auto matchers = getMatchers(store);
 
         Installables installables;
-        std::vector<size_t> indices;
+        std::vector<ProfileElement *> elems;
 
         auto matchedCount = 0;
         auto upgradedCount = 0;
 
-        for (size_t i = 0; i < manifest.elements.size(); ++i) {
-            auto & element(manifest.elements[i]);
-            if (!matches(*store, element, matchers)) {
+        for (auto & [name, element] : manifest.elements) {
+            if (!matches(*store, name, element, matchers)) {
                 continue;
             }
 
@@ -637,7 +639,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
             };
 
             installables.push_back(installable);
-            indices.push_back(i);
+            elems.push_back(&element);
         }
 
         if (upgradedCount == 0) {
@@ -661,7 +663,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
 
         for (size_t i = 0; i < installables.size(); ++i) {
             auto & installable = installables.at(i);
-            auto & element = manifest.elements[indices.at(i)];
+            auto & element = *elems.at(i);
             element.updateStorePaths(
                 getEvalStore(),
                 store,
@@ -693,11 +695,11 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro
         if (json) {
             std::cout << manifest.toJSON(*store).dump() << "\n";
         } else {
-            for (size_t i = 0; i < manifest.elements.size(); ++i) {
-                auto & element(manifest.elements[i]);
+            for (const auto & [i, e] : enumerate(manifest.elements)) {
+                auto & [name, element] = e;
                 if (i) logger->cout("");
                 logger->cout("Name:               " ANSI_BOLD "%s" ANSI_NORMAL "%s",
-                    element.name,
+                    name,
                     element.active ? "" : " " ANSI_RED "(inactive)" ANSI_NORMAL);
                 if (element.source) {
                     logger->cout("Flake attribute:    %s%s", element.source->attrPath, element.source->outputs.to_string());
diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh
index 618b6241d..003af5174 100644
--- a/tests/functional/nix-profile.sh
+++ b/tests/functional/nix-profile.sh
@@ -59,7 +59,7 @@ nix profile diff-closures | grep 'env-manifest.nix: ε → ∅'
 
 # Test XDG Base Directories support
 export NIX_CONFIG="use-xdg-base-directories = true"
-nix profile remove flake1
+nix profile remove flake1 2>&1 | grep 'removed 1 packages'
 nix profile install $flake1Dir
 [[ $($TEST_HOME/.local/state/nix/profile/bin/hello) = "Hello World" ]]
 unset NIX_CONFIG
@@ -80,7 +80,7 @@ nix profile rollback
 
 # Test uninstall.
 [ -e $TEST_HOME/.nix-profile/bin/foo ]
-nix profile remove foo
+nix profile remove foo 2>&1 | grep 'removed 1 packages'
 (! [ -e $TEST_HOME/.nix-profile/bin/foo ])
 nix profile history | grep 'foo: 1.0 -> ∅'
 nix profile diff-closures | grep 'Version 3 -> 4'
@@ -88,7 +88,7 @@ nix profile diff-closures | grep 'Version 3 -> 4'
 # Test installing a non-flake package.
 nix profile install --file ./simple.nix ''
 [[ $(cat $TEST_HOME/.nix-profile/hello) = "Hello World!" ]]
-nix profile remove simple
+nix profile remove simple 2>&1 | grep 'removed 1 packages'
 nix profile install $(nix-build --no-out-link ./simple.nix)
 [[ $(cat $TEST_HOME/.nix-profile/hello) = "Hello World!" ]]
 
@@ -96,8 +96,9 @@ nix profile install $(nix-build --no-out-link ./simple.nix)
 mkdir $TEST_ROOT/simple-too
 cp ./simple.nix ./config.nix simple.builder.sh $TEST_ROOT/simple-too
 nix profile install --file $TEST_ROOT/simple-too/simple.nix ''
-nix profile list | grep -A4 'Name:.*simple' | grep 'Name:.*simple1'
-nix profile remove simple1
+nix profile list | grep -A4 'Name:.*simple' | grep 'Name:.*simple-1'
+nix profile remove simple 2>&1 | grep 'removed 1 packages'
+nix profile remove simple-1 2>&1 | grep 'removed 1 packages'
 
 # Test wipe-history.
 nix profile wipe-history
@@ -110,7 +111,7 @@ nix profile upgrade flake1
 nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 3.0, 3.0-man"
 
 # Test new install of CA package.
-nix profile remove flake1
+nix profile remove flake1 2>&1 | grep 'removed 1 packages'
 printf 4.0 > $flake1Dir/version
 printf Utrecht > $flake1Dir/who
 nix profile install $flake1Dir
@@ -131,14 +132,14 @@ nix profile upgrade flake1
 [ -e $TEST_HOME/.nix-profile/share/man ]
 [ -e $TEST_HOME/.nix-profile/include ]
 
-nix profile remove flake1
+nix profile remove flake1 2>&1 | grep 'removed 1 packages'
 nix profile install "$flake1Dir^man"
 (! [ -e $TEST_HOME/.nix-profile/bin/hello ])
 [ -e $TEST_HOME/.nix-profile/share/man ]
 (! [ -e $TEST_HOME/.nix-profile/include ])
 
 # test priority
-nix profile remove flake1
+nix profile remove flake1 2>&1 | grep 'removed 1 packages'
 
 # Make another flake.
 flake2Dir=$TEST_ROOT/flake2

From a748e88bf4cca0fdc6ce75188e88017a7899d16b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 22 Dec 2023 16:27:31 +0100
Subject: [PATCH 286/421] nix profile: Remove check for "name" attribute in
 manifests

AFAIK, we've never emitted this attribute.
---
 src/nix/profile.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 8b3918b80..1b0c333bd 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -155,8 +155,6 @@ struct ProfileManifest
                 std::string name =
                     elems.is_object()
                     ? elem.key()
-                    : e.contains("name")
-                    ? (std::string) e["name"]
                     : element.source
                     ? getNameFromURL(parseURL(element.source->to_string())).value_or(element.identifier())
                     : element.identifier();

From 936a3642264ac159f3f9093710be3465b70e0e89 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 22 Dec 2023 16:35:58 +0100
Subject: [PATCH 287/421] getNameFromURL(): Support uppercase characters in
 attribute names

In particular, this makes it handle 'legacyPackages' correctly.
---
 src/libexpr/flake/url-name.cc        | 2 +-
 tests/unit/libexpr/flake/url-name.cc | 8 +++++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/libexpr/flake/url-name.cc b/src/libexpr/flake/url-name.cc
index 7e51aa2e1..753f197d5 100644
--- a/src/libexpr/flake/url-name.cc
+++ b/src/libexpr/flake/url-name.cc
@@ -4,7 +4,7 @@
 
 namespace nix {
 
-static const std::string attributeNamePattern("[a-z0-9_-]+");
+static const std::string attributeNamePattern("[a-zA-Z0-9_-]+");
 static const std::regex lastAttributeRegex("(?:" + attributeNamePattern + "\\.)*(?!default)(" + attributeNamePattern +")(\\^.*)?");
 static const std::string pathSegmentPattern("[a-zA-Z0-9_-]+");
 static const std::regex lastPathSegmentRegex(".*/(" + pathSegmentPattern +")");
diff --git a/tests/unit/libexpr/flake/url-name.cc b/tests/unit/libexpr/flake/url-name.cc
index 84d32837c..85387b323 100644
--- a/tests/unit/libexpr/flake/url-name.cc
+++ b/tests/unit/libexpr/flake/url-name.cc
@@ -5,11 +5,13 @@ namespace nix {
 
 /* ----------- tests for url-name.hh --------------------------------------------------*/
 
-    TEST(getNameFromURL, getsNameFromURL) {
+    TEST(getNameFromURL, getNameFromURL) {
         ASSERT_EQ(getNameFromURL(parseURL("path:/home/user/project")), "project");
         ASSERT_EQ(getNameFromURL(parseURL("path:~/repos/nixpkgs#packages.x86_64-linux.hello")), "hello");
-        ASSERT_EQ(getNameFromURL(parseURL("path:.#nonStandardAttr.mylaptop")), "nonStandardAttr.mylaptop");
-        ASSERT_EQ(getNameFromURL(parseURL("path:./repos/myflake#nonStandardAttr.mylaptop")), "nonStandardAttr.mylaptop");
+        ASSERT_EQ(getNameFromURL(parseURL("path:~/repos/nixpkgs#legacyPackages.x86_64-linux.hello")), "hello");
+        ASSERT_EQ(getNameFromURL(parseURL("path:~/repos/nixpkgs#packages.x86_64-linux.Hello")), "Hello");
+        ASSERT_EQ(getNameFromURL(parseURL("path:.#nonStandardAttr.mylaptop")), "mylaptop");
+        ASSERT_EQ(getNameFromURL(parseURL("path:./repos/myflake#nonStandardAttr.mylaptop")), "mylaptop");
         ASSERT_EQ(getNameFromURL(parseURL("path:./nixpkgs#packages.x86_64-linux.complex^bin,man")), "complex");
         ASSERT_EQ(getNameFromURL(parseURL("path:./myproj#packages.x86_64-linux.default^*")), "myproj");
 

From 9cb287657bec5a969d8bb1678d598d9fa820e60b Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Sat, 23 Dec 2023 17:15:09 -0500
Subject: [PATCH 288/421] remote-store test: Break out IFD expression into a
 separate file

---
 tests/functional/ifd.nix         | 10 ++++++++++
 tests/functional/remote-store.sh | 13 +------------
 2 files changed, 11 insertions(+), 12 deletions(-)
 create mode 100644 tests/functional/ifd.nix

diff --git a/tests/functional/ifd.nix b/tests/functional/ifd.nix
new file mode 100644
index 000000000..d0b9b54ad
--- /dev/null
+++ b/tests/functional/ifd.nix
@@ -0,0 +1,10 @@
+with import ./config.nix;
+import (
+  mkDerivation {
+    name = "foo";
+    bla = import ./dependencies.nix {};
+    buildCommand = "
+      echo \\\"hi\\\" > $out
+    ";
+  }
+)
diff --git a/tests/functional/remote-store.sh b/tests/functional/remote-store.sh
index 5c7bfde46..dc80f8b55 100644
--- a/tests/functional/remote-store.sh
+++ b/tests/functional/remote-store.sh
@@ -19,18 +19,7 @@ else
 fi
 
 # Test import-from-derivation through the daemon.
-[[ $(nix eval --impure --raw --expr '
-  with import ./config.nix;
-  import (
-    mkDerivation {
-      name = "foo";
-      bla = import ./dependencies.nix {};
-      buildCommand = "
-        echo \\\"hi\\\" > $out
-      ";
-    }
-  )
-') = hi ]]
+[[ $(nix eval --impure --raw --file ./ifd.nix) = hi ]]
 
 storeCleared=1 NIX_REMOTE_=$NIX_REMOTE $SHELL ./user-envs.sh
 

From c3942ef85ffbd83391410fbf012f1de366d2463c Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Sat, 23 Dec 2023 21:26:12 -0500
Subject: [PATCH 289/421] Build IFD in the build store when using eval-store.

Previously, IFDs would be built within the eval store, even though one
is typically using `--eval-store` precisely to *avoid* local builds.

Because the resulting Nix expression must be copied back to the eval
store in order to be imported, this requires the eval store to trust
the build store's signatures.
---
 doc/manual/rl-next/ifd-eval-store.md |  8 ++++++++
 src/libexpr/primops.cc               | 19 +++++++++++++------
 tests/functional/eval-store.sh       |  8 ++++++++
 3 files changed, 29 insertions(+), 6 deletions(-)
 create mode 100644 doc/manual/rl-next/ifd-eval-store.md

diff --git a/doc/manual/rl-next/ifd-eval-store.md b/doc/manual/rl-next/ifd-eval-store.md
new file mode 100644
index 000000000..835e7e7a3
--- /dev/null
+++ b/doc/manual/rl-next/ifd-eval-store.md
@@ -0,0 +1,8 @@
+---
+synopsis: import-from-derivation builds the derivation in the build store
+prs: 9661
+---
+
+When using `--eval-store`, `import`ing from a derivation will now result in the derivation being built on the build store, i.e. the store specified in the `store` Nix option.
+
+Because the resulting Nix expression must be copied back to the eval store in order to be imported, this requires the eval store to trust the build store's signatures.
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index a1502da45..58826b3bd 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -84,14 +84,14 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
     /* Build/substitute the context. */
     std::vector<DerivedPath> buildReqs;
     for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d });
-    store->buildPaths(buildReqs);
+    buildStore->buildPaths(buildReqs, bmNormal, store);
+
+    StorePathSet outputsToCopyAndAllow;
 
     for (auto & drv : drvs) {
-        auto outputs = resolveDerivedPath(*store, drv);
+        auto outputs = resolveDerivedPath(*buildStore, drv, &*store);
         for (auto & [outputName, outputPath] : outputs) {
-            /* Add the output of this derivations to the allowed
-               paths. */
-            allowPath(store->toRealPath(outputPath));
+            outputsToCopyAndAllow.insert(outputPath);
 
             /* Get all the output paths corresponding to the placeholders we had */
             if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
@@ -101,12 +101,19 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
                             .drvPath = drv.drvPath,
                             .output = outputName,
                         }).render(),
-                    store->printStorePath(outputPath)
+                    buildStore->printStorePath(outputPath)
                 );
             }
         }
     }
 
+    if (store != buildStore) copyClosure(*buildStore, *store, outputsToCopyAndAllow);
+    for (auto & outputPath : outputsToCopyAndAllow) {
+        /* Add the output of this derivations to the allowed
+           paths. */
+        allowPath(store->toRealPath(outputPath));
+    }
+
     return res;
 }
 
diff --git a/tests/functional/eval-store.sh b/tests/functional/eval-store.sh
index ec99fd953..9937ecbce 100644
--- a/tests/functional/eval-store.sh
+++ b/tests/functional/eval-store.sh
@@ -40,3 +40,11 @@ if [[ ! -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then
     (! ls $NIX_STORE_DIR/*.drv)
 fi
 ls $eval_store/nix/store/*.drv
+
+clearStore
+rm -rf "$eval_store"
+
+# Confirm that import-from-derivation builds on the build store
+[[ $(nix eval --eval-store "$eval_store?require-sigs=false" --impure --raw --file ./ifd.nix) = hi ]]
+ls $NIX_STORE_DIR/*dependencies-top/foobar
+(! ls $eval_store/nix/store/*dependencies-top/foobar)

From e2399fc94935c9bc1ae6670c5d445214e039ac84 Mon Sep 17 00:00:00 2001
From: Brian Le <brian.mk.le@gmail.com>
Date: Tue, 26 Dec 2023 17:12:28 -0500
Subject: [PATCH 290/421] Change "dervation" typos to "derivation"

---
 doc/manual/src/language/derivations.md | 2 +-
 src/libstore/remote-store.cc           | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md
index 2aded5527..cbb30d074 100644
--- a/doc/manual/src/language/derivations.md
+++ b/doc/manual/src/language/derivations.md
@@ -274,7 +274,7 @@ The [`builder`](#attr-builder) is executed as follows:
     directory (typically, `/nix/store`).
 
   - `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs`
-    is set to `true` for the dervation. A detailed explanation of this
+    is set to `true` for the derivation. A detailed explanation of this
     behavior can be found in the
     [section about structured attrs](./advanced-attributes.md#adv-attr-structuredAttrs).
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 4d0113594..f0df646ca 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -186,7 +186,7 @@ void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source,
                 if (m.find("parsing derivation") != std::string::npos &&
                     m.find("expected string") != std::string::npos &&
                     m.find("Derive([") != std::string::npos)
-                    throw Error("%s, this might be because the daemon is too old to understand dependencies on dynamic derivations. Check to see if the raw dervation is in the form '%s'", std::move(m), "DrvWithVersion(..)");
+                    throw Error("%s, this might be because the daemon is too old to understand dependencies on dynamic derivations. Check to see if the raw derivation is in the form '%s'", std::move(m), "DrvWithVersion(..)");
             }
             throw;
         }

From b6313f64f7be11e0fe74b17cb31dbbf12b2e7725 Mon Sep 17 00:00:00 2001
From: DavHau <hsngrmpf+github@gmail.com>
Date: Wed, 27 Dec 2023 19:57:27 +0700
Subject: [PATCH 291/421] saner default for log-lines: change to 25

This seems to be a much saner default. 10 lines are just not enough in so many cases.
---
 src/libstore/globals.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index b35dc37a1..c12998f8e 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -144,7 +144,7 @@ public:
      */
     bool verboseBuild = true;
 
-    Setting<size_t> logLines{this, 10, "log-lines",
+    Setting<size_t> logLines{this, 25, "log-lines",
         "The number of lines of the tail of "
         "the log to show if a build fails."};
 

From 99a691c8a1abffd30077bd5f005cb8d4bbafae5c Mon Sep 17 00:00:00 2001
From: pennae <github@quasiparticle.net>
Date: Sun, 24 Dec 2023 21:14:08 +0100
Subject: [PATCH 292/421] don't use istreams in hot paths
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

istream sentry objects are very expensive for single-character
operations, and since we don't configure exception masks for the
istreams used here they don't even do anything. all we need is
end-of-string checks and an advancing position in an immutable memory
buffer, both of which can be had for much cheaper than istreams allow.

the effect of this change is most apparent on empty stores.

before:

Benchmark 1: nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'
  Time (mean ± σ):      7.167 s ±  0.013 s    [User: 5.528 s, System: 1.431 s]
  Range (min … max):    7.147 s …  7.182 s    10 runs

after:

Benchmark 1: nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'
  Time (mean ± σ):      6.963 s ±  0.011 s    [User: 5.330 s, System: 1.421 s]
  Range (min … max):    6.943 s …  6.974 s    10 runs
---
 src/libstore/derivations.cc | 47 +++++++++++++++++++++++++++----------
 1 file changed, 34 insertions(+), 13 deletions(-)

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 8a7d660ff..973ce5211 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -154,18 +154,39 @@ StorePath writeDerivation(Store & store,
 }
 
 
-/* Read string `s' from stream `str'. */
-static void expect(std::istream & str, std::string_view s)
-{
-    for (auto & c : s) {
-        if (str.get() != c)
-            throw FormatError("expected string '%1%'", s);
+namespace {
+/**
+ * This mimics std::istream to some extent. We use this much smaller implementation
+ * instead of plain istreams because the sentry object overhead is too high.
+ */
+struct StringViewStream {
+    std::string_view remaining;
+
+    int peek() const {
+        return remaining.empty() ? EOF : remaining[0];
     }
+
+    int get() {
+        if (remaining.empty()) return EOF;
+        char c = remaining[0];
+        remaining.remove_prefix(1);
+        return c;
+    }
+};
+}
+
+
+/* Read string `s' from stream `str'. */
+static void expect(StringViewStream & str, std::string_view s)
+{
+    if (!str.remaining.starts_with(s))
+        throw FormatError("expected string '%1%'", s);
+    str.remaining.remove_prefix(s.size());
 }
 
 
 /* Read a C-style string from stream `str'. */
-static std::string parseString(std::istream & str)
+static std::string parseString(StringViewStream & str)
 {
     std::string res;
     expect(str, "\"");
@@ -187,7 +208,7 @@ static void validatePath(std::string_view s) {
         throw FormatError("bad path '%1%' in derivation", s);
 }
 
-static Path parsePath(std::istream & str)
+static Path parsePath(StringViewStream & str)
 {
     auto s = parseString(str);
     validatePath(s);
@@ -195,7 +216,7 @@ static Path parsePath(std::istream & str)
 }
 
 
-static bool endOfList(std::istream & str)
+static bool endOfList(StringViewStream & str)
 {
     if (str.peek() == ',') {
         str.get();
@@ -209,7 +230,7 @@ static bool endOfList(std::istream & str)
 }
 
 
-static StringSet parseStrings(std::istream & str, bool arePaths)
+static StringSet parseStrings(StringViewStream & str, bool arePaths)
 {
     StringSet res;
     expect(str, "[");
@@ -267,7 +288,7 @@ static DerivationOutput parseDerivationOutput(
 }
 
 static DerivationOutput parseDerivationOutput(
-    const StoreDirConfig & store, std::istringstream & str,
+    const StoreDirConfig & store, StringViewStream & str,
     const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings)
 {
     expect(str, ","); const auto pathS = parseString(str);
@@ -297,7 +318,7 @@ enum struct DerivationATermVersion {
 
 static DerivedPathMap<StringSet>::ChildNode parseDerivedPathMapNode(
     const StoreDirConfig & store,
-    std::istringstream & str,
+    StringViewStream & str,
     DerivationATermVersion version)
 {
     DerivedPathMap<StringSet>::ChildNode node;
@@ -349,7 +370,7 @@ Derivation parseDerivation(
     Derivation drv;
     drv.name = name;
 
-    std::istringstream str(std::move(s));
+    StringViewStream str{s};
     expect(str, "D");
     DerivationATermVersion version;
     switch (str.peek()) {

From 2cfc4ace35d1c8cca917c487be3cfddfcf3bba01 Mon Sep 17 00:00:00 2001
From: pennae <github@quasiparticle.net>
Date: Tue, 26 Dec 2023 17:40:55 +0100
Subject: [PATCH 293/421] malloc/memset even less
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

more buffers that can be uninitialized and on the stack. small
difference, but still worth doing.

before:

Benchmark 1: nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'
  Time (mean ± σ):      6.963 s ±  0.011 s    [User: 5.330 s, System: 1.421 s]
  Range (min … max):    6.943 s …  6.974 s    10 runs

after:

Benchmark 1: nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'
  Time (mean ± σ):      6.952 s ±  0.015 s    [User: 5.294 s, System: 1.452 s]
  Range (min … max):    6.926 s …  6.974 s    10 runs
---
 src/libutil/archive.cc     | 2 +-
 src/libutil/file-system.cc | 2 +-
 src/libutil/serialise.cc   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index 465df2073..712ea51c7 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -140,7 +140,7 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path)
     sink.preallocateContents(size);
 
     uint64_t left = size;
-    std::vector<char> buf(65536);
+    std::array<char, 65536> buf;
 
     while (left) {
         checkInterrupt();
diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc
index c96effff9..4cac35ace 100644
--- a/src/libutil/file-system.cc
+++ b/src/libutil/file-system.cc
@@ -307,7 +307,7 @@ void writeFile(const Path & path, Source & source, mode_t mode, bool sync)
     if (!fd)
         throw SysError("opening file '%1%'", path);
 
-    std::vector<char> buf(64 * 1024);
+    std::array<char, 64 * 1024> buf;
 
     try {
         while (true) {
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index f465bd0de..76b378e18 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -82,7 +82,7 @@ void Source::operator () (std::string_view data)
 void Source::drainInto(Sink & sink)
 {
     std::string s;
-    std::vector<char> buf(8192);
+    std::array<char, 8192> buf;
     while (true) {
         size_t n;
         try {

From 7434caca0545bd6194bb52eebf6fdf0424755eb0 Mon Sep 17 00:00:00 2001
From: Rebecca Turner <rbt@sent.as>
Date: Fri, 15 Dec 2023 11:52:21 -0800
Subject: [PATCH 294/421] Fix segfault on infinite recursion in some cases
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This fixes a segfault on infinite function call recursion (rather than
infinite thunk recursion) by tracking the function call depth in
`EvalState`.

Additionally, to avoid printing extremely long stack traces, stack
frames are now deduplicated, with a `(19997 duplicate traces omitted)`
message. This should only really be triggered in infinite recursion
scenarios.

Before:

    $ nix-instantiate --eval --expr '(x: x x) (x: x x)'
    Segmentation fault: 11

After:

    $ nix-instantiate --eval --expr '(x: x x) (x: x x)'
    error: stack overflow

           at «string»:1:14:
                1| (x: x x) (x: x x)
                 |              ^

    $ nix-instantiate --eval --expr '(x: x x) (x: x x)' --show-trace
    error:
           … from call site
             at «string»:1:1:
                1| (x: x x) (x: x x)
                 | ^

           … while calling anonymous lambda
             at «string»:1:2:
                1| (x: x x) (x: x x)
                 |  ^

           … from call site
             at «string»:1:5:
                1| (x: x x) (x: x x)
                 |     ^

           … while calling anonymous lambda
             at «string»:1:11:
                1| (x: x x) (x: x x)
                 |           ^

           … from call site
             at «string»:1:14:
                1| (x: x x) (x: x x)
                 |              ^

           (19997 duplicate traces omitted)

           error: stack overflow
           at «string»:1:14:
                1| (x: x x) (x: x x)
                 |              ^
---
 .../rl-next/stack-overflow-segfaults.md       |  32 +++++
 src/libexpr/eval-settings.hh                  |   3 +
 src/libexpr/eval.cc                           |  18 +++
 src/libexpr/eval.hh                           |   5 +
 src/libutil/error.cc                          | 111 +++++++++++++++++-
 src/libutil/error.hh                          |   8 ++
 .../lang/eval-fail-duplicate-traces.err.exp   |  44 +++++++
 .../lang/eval-fail-duplicate-traces.nix       |   9 ++
 ...val-fail-infinite-recursion-lambda.err.exp |  38 ++++++
 .../eval-fail-infinite-recursion-lambda.nix   |   1 +
 .../lang/eval-fail-mutual-recursion.err.exp   |  57 +++++++++
 .../lang/eval-fail-mutual-recursion.nix       |  36 ++++++
 12 files changed, 358 insertions(+), 4 deletions(-)
 create mode 100644 doc/manual/rl-next/stack-overflow-segfaults.md
 create mode 100644 tests/functional/lang/eval-fail-duplicate-traces.err.exp
 create mode 100644 tests/functional/lang/eval-fail-duplicate-traces.nix
 create mode 100644 tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp
 create mode 100644 tests/functional/lang/eval-fail-infinite-recursion-lambda.nix
 create mode 100644 tests/functional/lang/eval-fail-mutual-recursion.err.exp
 create mode 100644 tests/functional/lang/eval-fail-mutual-recursion.nix

diff --git a/doc/manual/rl-next/stack-overflow-segfaults.md b/doc/manual/rl-next/stack-overflow-segfaults.md
new file mode 100644
index 000000000..3d9753248
--- /dev/null
+++ b/doc/manual/rl-next/stack-overflow-segfaults.md
@@ -0,0 +1,32 @@
+---
+synopsis: Some stack overflow segfaults are fixed
+issues: 9616
+prs: 9617
+---
+
+The number of nested function calls has been restricted, to detect and report
+infinite function call recursions. The default maximum call depth is 10,000 and
+can be set with [the `max-call-depth`
+option](@docroot@/command-ref/conf-file.md#conf-max-call-depth).
+
+This fixes segfaults or the following unhelpful error message in many cases:
+
+    error: stack overflow (possible infinite recursion)
+
+Before:
+
+```
+$ nix-instantiate --eval --expr '(x: x x) (x: x x)'
+Segmentation fault: 11
+```
+
+After:
+
+```
+$ nix-instantiate --eval --expr '(x: x x) (x: x x)'
+error: stack overflow
+
+       at «string»:1:14:
+            1| (x: x x) (x: x x)
+             |              ^
+```
diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh
index ad187ca01..2f6c12d45 100644
--- a/src/libexpr/eval-settings.hh
+++ b/src/libexpr/eval-settings.hh
@@ -124,6 +124,9 @@ struct EvalSettings : Config
 
     Setting<bool> traceVerbose{this, false, "trace-verbose",
         "Whether `builtins.traceVerbose` should trace its first argument when evaluated."};
+
+    Setting<unsigned int> maxCallDepth{this, 10000, "max-call-depth",
+        "The maximum function call depth to allow before erroring."};
 };
 
 extern EvalSettings evalSettings;
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 810843995..f73e22ba0 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1505,9 +1505,27 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v)
     v.mkLambda(&env, this);
 }
 
+namespace {
+/** Increments a count on construction and decrements on destruction.
+ */
+class CallDepth {
+  size_t & count;
+public:
+  CallDepth(size_t & count) : count(count) {
+    ++count;
+  }
+  ~CallDepth() {
+    --count;
+  }
+};
+};
 
 void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos)
 {
+    if (callDepth > evalSettings.maxCallDepth)
+        error("stack overflow; max-call-depth exceeded").atPos(pos).template debugThrow<EvalError>();
+    CallDepth _level(callDepth);
+
     auto trace = evalSettings.traceFunctionCalls
         ? std::make_unique<FunctionCallTrace>(positions[pos])
         : nullptr;
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index da2d256db..7dbffe38c 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -622,6 +622,11 @@ private:
         const SourcePath & basePath,
         std::shared_ptr<StaticEnv> & staticEnv);
 
+    /**
+     * Current Nix call stack depth, used with `max-call-depth` setting to throw stack overflow hopefully before we run out of system stack.
+     */
+    size_t callDepth = 0;
+
 public:
 
     /**
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index bc0194d59..e42925c2b 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -50,6 +50,32 @@ std::ostream & operator <<(std::ostream & str, const AbstractPos & pos)
     return str;
 }
 
+/**
+ * An arbitrarily defined value comparison for the purpose of using traces in the key of a sorted container.
+ */
+inline bool operator<(const Trace& lhs, const Trace& rhs)
+{
+    // `std::shared_ptr` does not have value semantics for its comparison
+    // functions, so we need to check for nulls and compare the dereferenced
+    // values here.
+    if (lhs.pos != rhs.pos) {
+        if (!lhs.pos)
+            return true;
+        if (!rhs.pos)
+            return false;
+        if (*lhs.pos != *rhs.pos)
+            return *lhs.pos < *rhs.pos;
+    }
+    // This formats a freshly formatted hint string and then throws it away, which
+    // shouldn't be much of a problem because it only runs when pos is equal, and this function is
+    // used for trace printing, which is infrequent.
+    return std::forward_as_tuple(lhs.hint.str(), lhs.frame)
+        < std::forward_as_tuple(rhs.hint.str(), rhs.frame);
+}
+inline bool operator> (const Trace& lhs, const Trace& rhs) { return rhs < lhs; }
+inline bool operator<=(const Trace& lhs, const Trace& rhs) { return !(lhs > rhs); }
+inline bool operator>=(const Trace& lhs, const Trace& rhs) { return !(lhs < rhs); }
+
 std::optional<LinesOfCode> AbstractPos::getCodeLines() const
 {
     if (line == 0)
@@ -185,6 +211,69 @@ static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std
     return hasPos;
 }
 
+void printTrace(
+    std::ostream & output,
+    const std::string_view & indent,
+    size_t & count,
+    const Trace & trace)
+{
+    output << "\n" << "… " << trace.hint.str() << "\n";
+
+    if (printPosMaybe(output, indent, trace.pos))
+        count++;
+}
+
+void printSkippedTracesMaybe(
+    std::ostream & output,
+    const std::string_view & indent,
+    size_t & count,
+    std::vector<Trace> & skippedTraces,
+    std::set<Trace> tracesSeen)
+{
+    if (skippedTraces.size() > 0) {
+        // If we only skipped a few frames, print them out normally;
+        // messages like "1 duplicate frames omitted" aren't helpful.
+        if (skippedTraces.size() <= 5) {
+            for (auto & trace : skippedTraces) {
+                printTrace(output, indent, count, trace);
+            }
+        } else {
+            output << "\n" << ANSI_WARNING "(" << skippedTraces.size() << " duplicate frames omitted)" ANSI_NORMAL << "\n";
+            // Clear the set of "seen" traces after printing a chunk of
+            // `duplicate frames omitted`.
+            //
+            // Consider a mutually recursive stack trace with:
+            // - 10 entries of A
+            // - 10 entries of B
+            // - 10 entries of A
+            //
+            // If we don't clear `tracesSeen` here, we would print output like this:
+            // - 1 entry of A
+            // - (9 duplicate frames omitted)
+            // - 1 entry of B
+            // - (19 duplicate frames omitted)
+            //
+            // This would obscure the control flow, which went from A,
+            // to B, and back to A again.
+            //
+            // In contrast, if we do clear `tracesSeen`, the output looks like this:
+            // - 1 entry of A
+            // - (9 duplicate frames omitted)
+            // - 1 entry of B
+            // - (9 duplicate frames omitted)
+            // - 1 entry of A
+            // - (9 duplicate frames omitted)
+            //
+            // See: `tests/functional/lang/eval-fail-mutual-recursion.nix`
+            tracesSeen.clear();
+        }
+    }
+    // We've either printed each trace in `skippedTraces` normally, or
+    // printed a chunk of `duplicate frames omitted`. Either way, we've
+    // processed these traces and can clear them.
+    skippedTraces.clear();
+}
+
 std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace)
 {
     std::string prefix;
@@ -333,7 +422,13 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
 
     bool frameOnly = false;
     if (!einfo.traces.empty()) {
+        // Stack traces seen since we last printed a chunk of `duplicate frames
+        // omitted`.
+        std::set<Trace> tracesSeen;
+        // A consecutive sequence of stack traces that are all in `tracesSeen`.
+        std::vector<Trace> skippedTraces;
         size_t count = 0;
+
         for (const auto & trace : einfo.traces) {
             if (trace.hint.str().empty()) continue;
             if (frameOnly && !trace.frame) continue;
@@ -343,14 +438,21 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
                 break;
             }
 
+            if (tracesSeen.count(trace)) {
+                skippedTraces.push_back(trace);
+                continue;
+            }
+            tracesSeen.insert(trace);
+
+            printSkippedTracesMaybe(oss, ellipsisIndent, count, skippedTraces, tracesSeen);
+
             count++;
             frameOnly = trace.frame;
 
-            oss << "\n" << "… " << trace.hint.str() << "\n";
-
-            if (printPosMaybe(oss, ellipsisIndent, trace.pos))
-                count++;
+            printTrace(oss, ellipsisIndent, count, trace);
         }
+
+        printSkippedTracesMaybe(oss, ellipsisIndent, count, skippedTraces, tracesSeen);
         oss << "\n" << prefix;
     }
 
@@ -369,4 +471,5 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
 
     return out;
 }
+
 }
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index c04dcbd77..baffca128 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -25,6 +25,7 @@
 #include <memory>
 #include <map>
 #include <optional>
+#include <compare>
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -88,6 +89,8 @@ struct AbstractPos
     std::optional<LinesOfCode> getCodeLines() const;
 
     virtual ~AbstractPos() = default;
+
+    inline auto operator<=>(const AbstractPos& rhs) const = default;
 };
 
 std::ostream & operator << (std::ostream & str, const AbstractPos & pos);
@@ -103,6 +106,11 @@ struct Trace {
     bool frame;
 };
 
+inline bool operator<(const Trace& lhs, const Trace& rhs);
+inline bool operator> (const Trace& lhs, const Trace& rhs);
+inline bool operator<=(const Trace& lhs, const Trace& rhs);
+inline bool operator>=(const Trace& lhs, const Trace& rhs);
+
 struct ErrorInfo {
     Verbosity level;
     hintformat msg;
diff --git a/tests/functional/lang/eval-fail-duplicate-traces.err.exp b/tests/functional/lang/eval-fail-duplicate-traces.err.exp
new file mode 100644
index 000000000..32ad9b376
--- /dev/null
+++ b/tests/functional/lang/eval-fail-duplicate-traces.err.exp
@@ -0,0 +1,44 @@
+error:
+       … from call site
+         at /pwd/lang/eval-fail-duplicate-traces.nix:9:3:
+            8| in
+            9|   throwAfter 2
+             |   ^
+           10|
+
+       … while calling 'throwAfter'
+         at /pwd/lang/eval-fail-duplicate-traces.nix:4:16:
+            3| let
+            4|   throwAfter = n:
+             |                ^
+            5|     if n > 0
+
+       … from call site
+         at /pwd/lang/eval-fail-duplicate-traces.nix:6:10:
+            5|     if n > 0
+            6|     then throwAfter (n - 1)
+             |          ^
+            7|     else throw "Uh oh!";
+
+       … while calling 'throwAfter'
+         at /pwd/lang/eval-fail-duplicate-traces.nix:4:16:
+            3| let
+            4|   throwAfter = n:
+             |                ^
+            5|     if n > 0
+
+       … from call site
+         at /pwd/lang/eval-fail-duplicate-traces.nix:6:10:
+            5|     if n > 0
+            6|     then throwAfter (n - 1)
+             |          ^
+            7|     else throw "Uh oh!";
+
+       … while calling 'throwAfter'
+         at /pwd/lang/eval-fail-duplicate-traces.nix:4:16:
+            3| let
+            4|   throwAfter = n:
+             |                ^
+            5|     if n > 0
+
+       error: Uh oh!
diff --git a/tests/functional/lang/eval-fail-duplicate-traces.nix b/tests/functional/lang/eval-fail-duplicate-traces.nix
new file mode 100644
index 000000000..17ce374ec
--- /dev/null
+++ b/tests/functional/lang/eval-fail-duplicate-traces.nix
@@ -0,0 +1,9 @@
+# Check that we only omit duplicate stack traces when there's a bunch of them.
+# Here, there's only a couple duplicate entries, so we output them all.
+let
+  throwAfter = n:
+    if n > 0
+    then throwAfter (n - 1)
+    else throw "Uh oh!";
+in
+  throwAfter 2
diff --git a/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp b/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp
new file mode 100644
index 000000000..5d843d827
--- /dev/null
+++ b/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp
@@ -0,0 +1,38 @@
+error:
+       … from call site
+         at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:1:
+            1| (x: x x) (x: x x)
+             | ^
+            2|
+
+       … while calling anonymous lambda
+         at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:2:
+            1| (x: x x) (x: x x)
+             |  ^
+            2|
+
+       … from call site
+         at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:5:
+            1| (x: x x) (x: x x)
+             |     ^
+            2|
+
+       … while calling anonymous lambda
+         at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:11:
+            1| (x: x x) (x: x x)
+             |           ^
+            2|
+
+       … from call site
+         at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:14:
+            1| (x: x x) (x: x x)
+             |              ^
+            2|
+
+       (19997 duplicate frames omitted)
+
+       error: stack overflow; max-call-depth exceeded
+       at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:14:
+            1| (x: x x) (x: x x)
+             |              ^
+            2|
diff --git a/tests/functional/lang/eval-fail-infinite-recursion-lambda.nix b/tests/functional/lang/eval-fail-infinite-recursion-lambda.nix
new file mode 100644
index 000000000..dd0a8bf2e
--- /dev/null
+++ b/tests/functional/lang/eval-fail-infinite-recursion-lambda.nix
@@ -0,0 +1 @@
+(x: x x) (x: x x)
diff --git a/tests/functional/lang/eval-fail-mutual-recursion.err.exp b/tests/functional/lang/eval-fail-mutual-recursion.err.exp
new file mode 100644
index 000000000..dc2e11766
--- /dev/null
+++ b/tests/functional/lang/eval-fail-mutual-recursion.err.exp
@@ -0,0 +1,57 @@
+error:
+       … from call site
+         at /pwd/lang/eval-fail-mutual-recursion.nix:36:3:
+           35| in
+           36|   throwAfterA true 10
+             |   ^
+           37|
+
+       … while calling 'throwAfterA'
+         at /pwd/lang/eval-fail-mutual-recursion.nix:29:26:
+           28|
+           29|   throwAfterA = recurse: n:
+             |                          ^
+           30|     if n > 0
+
+       … from call site
+         at /pwd/lang/eval-fail-mutual-recursion.nix:31:10:
+           30|     if n > 0
+           31|     then throwAfterA recurse (n - 1)
+             |          ^
+           32|     else if recurse
+
+       (19 duplicate frames omitted)
+
+       … from call site
+         at /pwd/lang/eval-fail-mutual-recursion.nix:33:10:
+           32|     else if recurse
+           33|     then throwAfterB true 10
+             |          ^
+           34|     else throw "Uh oh!";
+
+       … while calling 'throwAfterB'
+         at /pwd/lang/eval-fail-mutual-recursion.nix:22:26:
+           21| let
+           22|   throwAfterB = recurse: n:
+             |                          ^
+           23|     if n > 0
+
+       … from call site
+         at /pwd/lang/eval-fail-mutual-recursion.nix:24:10:
+           23|     if n > 0
+           24|     then throwAfterB recurse (n - 1)
+             |          ^
+           25|     else if recurse
+
+       (19 duplicate frames omitted)
+
+       … from call site
+         at /pwd/lang/eval-fail-mutual-recursion.nix:26:10:
+           25|     else if recurse
+           26|     then throwAfterA false 10
+             |          ^
+           27|     else throw "Uh oh!";
+
+       (21 duplicate frames omitted)
+
+       error: Uh oh!
diff --git a/tests/functional/lang/eval-fail-mutual-recursion.nix b/tests/functional/lang/eval-fail-mutual-recursion.nix
new file mode 100644
index 000000000..d090d3158
--- /dev/null
+++ b/tests/functional/lang/eval-fail-mutual-recursion.nix
@@ -0,0 +1,36 @@
+# Check that stack frame deduplication only affects consecutive intervals, and
+# that they are reported independently of any preceding sections, even if
+# they're indistinguishable.
+#
+# In terms of the current implementation, we check that we clear the set of
+# "seen frames" after eliding a group of frames.
+#
+# Suppose we have:
+# - 10 frames in a function A
+# - 10 frames in a function B
+# - 10 frames in a function A
+#
+# We want to output:
+# - a few frames of A (skip the rest)
+# - a few frames of B (skip the rest)
+# - a few frames of A (skip the rest)
+#
+# If we implemented this in the naive manner, we'd instead get:
+# - a few frames of A (skip the rest)
+# - a few frames of B (skip the rest, _and_ skip the remaining frames of A)
+let
+  throwAfterB = recurse: n:
+    if n > 0
+    then throwAfterB recurse (n - 1)
+    else if recurse
+    then throwAfterA false 10
+    else throw "Uh oh!";
+
+  throwAfterA = recurse: n:
+    if n > 0
+    then throwAfterA recurse (n - 1)
+    else if recurse
+    then throwAfterB true 10
+    else throw "Uh oh!";
+in
+  throwAfterA true 10

From 79d3d412cacd210bc9a0e9ba5407eea67c8e3868 Mon Sep 17 00:00:00 2001
From: pennae <github@quasiparticle.net>
Date: Tue, 26 Dec 2023 22:18:42 +0100
Subject: [PATCH 295/421] optimize derivation string parsing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

a bunch of derivation strings contain no escape sequences. we can
optimize for this fact by first scanning for the end of a derivation
string and simply returning the contents unmodified if no escape
sequences were found. to make this even more efficient we can also use
BackedStringViews to avoid copies, avoiding heap allocations for
transient data.

before:

Benchmark 1: nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'
  Time (mean ± σ):      6.952 s ±  0.015 s    [User: 5.294 s, System: 1.452 s]
  Range (min … max):    6.926 s …  6.974 s    10 runs

after:

Benchmark 1: nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'
  Time (mean ± σ):      6.907 s ±  0.012 s    [User: 5.272 s, System: 1.429 s]
  Range (min … max):    6.893 s …  6.926 s    10 runs
---
 doc/manual/rl-next/drv-string-parse-hang.md |  6 ++
 src/libstore/derivations.cc                 | 65 +++++++++++++--------
 2 files changed, 48 insertions(+), 23 deletions(-)
 create mode 100644 doc/manual/rl-next/drv-string-parse-hang.md

diff --git a/doc/manual/rl-next/drv-string-parse-hang.md b/doc/manual/rl-next/drv-string-parse-hang.md
new file mode 100644
index 000000000..1e041d3e9
--- /dev/null
+++ b/doc/manual/rl-next/drv-string-parse-hang.md
@@ -0,0 +1,6 @@
+---
+synopsis: Fix handling of truncated `.drv` files.
+prs: 9673
+---
+
+Previously a `.drv` that was truncated in the middle of a string would case nix to enter an infinite loop, eventually exhausting all memory and crashing.
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 973ce5211..89d902917 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -2,6 +2,7 @@
 #include "downstream-placeholder.hh"
 #include "store-api.hh"
 #include "globals.hh"
+#include "types.hh"
 #include "util.hh"
 #include "split.hh"
 #include "common-protocol.hh"
@@ -186,20 +187,38 @@ static void expect(StringViewStream & str, std::string_view s)
 
 
 /* Read a C-style string from stream `str'. */
-static std::string parseString(StringViewStream & str)
+static BackedStringView parseString(StringViewStream & str)
 {
-    std::string res;
     expect(str, "\"");
-    int c;
-    while ((c = str.get()) != '"')
-        if (c == '\\') {
-            c = str.get();
-            if (c == 'n') res += '\n';
-            else if (c == 'r') res += '\r';
-            else if (c == 't') res += '\t';
-            else res += c;
+    auto c = str.remaining.begin(), end = str.remaining.end();
+    bool escaped = false;
+    for (; c != end && *c != '"'; c++) {
+        if (*c == '\\') {
+            c++;
+            if (c == end)
+                throw FormatError("unterminated string in derivation");
+            escaped = true;
         }
-        else res += c;
+    }
+
+    const auto contentLen = c - str.remaining.begin();
+    const auto content = str.remaining.substr(0, contentLen);
+    str.remaining.remove_prefix(contentLen + 1);
+
+    if (!escaped)
+        return content;
+
+    std::string res;
+    res.reserve(content.size());
+    for (c = content.begin(), end = content.end(); c != end; c++)
+        if (*c == '\\') {
+            c++;
+            if (*c == 'n') res += '\n';
+            else if (*c == 'r') res += '\r';
+            else if (*c == 't') res += '\t';
+            else res += *c;
+        }
+        else res += *c;
     return res;
 }
 
@@ -210,7 +229,7 @@ static void validatePath(std::string_view s) {
 
 static Path parsePath(StringViewStream & str)
 {
-    auto s = parseString(str);
+    auto s = parseString(str).toOwned();
     validatePath(s);
     return s;
 }
@@ -235,7 +254,7 @@ static StringSet parseStrings(StringViewStream & str, bool arePaths)
     StringSet res;
     expect(str, "[");
     while (!endOfList(str))
-        res.insert(arePaths ? parsePath(str) : parseString(str));
+        res.insert(arePaths ? parsePath(str) : parseString(str).toOwned());
     return res;
 }
 
@@ -296,7 +315,7 @@ static DerivationOutput parseDerivationOutput(
     expect(str, ","); const auto hash = parseString(str);
     expect(str, ")");
 
-    return parseDerivationOutput(store, pathS, hashAlgo, hash, xpSettings);
+    return parseDerivationOutput(store, *pathS, *hashAlgo, *hash, xpSettings);
 }
 
 /**
@@ -344,7 +363,7 @@ static DerivedPathMap<StringSet>::ChildNode parseDerivedPathMapNode(
             expect(str, ",[");
             while (!endOfList(str)) {
                 expect(str, "(");
-                auto outputName = parseString(str);
+                auto outputName = parseString(str).toOwned();
                 expect(str, ",");
                 node.childMap.insert_or_assign(outputName, parseDerivedPathMapNode(store, str, version));
                 expect(str, ")");
@@ -381,12 +400,12 @@ Derivation parseDerivation(
     case 'r': {
         expect(str, "rvWithVersion(");
         auto versionS = parseString(str);
-        if (versionS == "xp-dyn-drv") {
+        if (*versionS == "xp-dyn-drv") {
             // Only verison we have so far
             version = DerivationATermVersion::DynamicDerivations;
             xpSettings.require(Xp::DynamicDerivations);
         } else {
-            throw FormatError("Unknown derivation ATerm format version '%s'", versionS);
+            throw FormatError("Unknown derivation ATerm format version '%s'", *versionS);
         }
         expect(str, ",");
         break;
@@ -398,7 +417,7 @@ Derivation parseDerivation(
     /* Parse the list of outputs. */
     expect(str, "[");
     while (!endOfList(str)) {
-        expect(str, "("); std::string id = parseString(str);
+        expect(str, "("); std::string id = parseString(str).toOwned();
         auto output = parseDerivationOutput(store, str, xpSettings);
         drv.outputs.emplace(std::move(id), std::move(output));
     }
@@ -414,19 +433,19 @@ Derivation parseDerivation(
     }
 
     expect(str, ","); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true));
-    expect(str, ","); drv.platform = parseString(str);
-    expect(str, ","); drv.builder = parseString(str);
+    expect(str, ","); drv.platform = parseString(str).toOwned();
+    expect(str, ","); drv.builder = parseString(str).toOwned();
 
     /* Parse the builder arguments. */
     expect(str, ",[");
     while (!endOfList(str))
-        drv.args.push_back(parseString(str));
+        drv.args.push_back(parseString(str).toOwned());
 
     /* Parse the environment variables. */
     expect(str, ",[");
     while (!endOfList(str)) {
-        expect(str, "("); auto name = parseString(str);
-        expect(str, ","); auto value = parseString(str);
+        expect(str, "("); auto name = parseString(str).toOwned();
+        expect(str, ","); auto value = parseString(str).toOwned();
         expect(str, ")");
         drv.env[name] = value;
     }

From 02c64abf1e892220cb62ce3b7e1598030fb6a61c Mon Sep 17 00:00:00 2001
From: pennae <github@quasiparticle.net>
Date: Tue, 26 Dec 2023 05:44:52 +0100
Subject: [PATCH 296/421] use translation table for drv string parsing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

the table is very small compared to cache sizes and a single indexed
load is faster than three comparisons.

before:

Benchmark 1: nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'
  Time (mean ± σ):      6.907 s ±  0.012 s    [User: 5.272 s, System: 1.429 s]
  Range (min … max):    6.893 s …  6.926 s    10 runs

after:

Benchmark 1: nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'
  Time (mean ± σ):      6.883 s ±  0.016 s    [User: 5.250 s, System: 1.424 s]
  Range (min … max):    6.860 s …  6.905 s    10 runs
---
 src/libstore/derivations.cc | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 89d902917..89a345057 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -174,6 +174,17 @@ struct StringViewStream {
         return c;
     }
 };
+
+constexpr struct Escapes {
+    char map[256];
+    constexpr Escapes() {
+        for (int i = 0; i < 256; i++) map[i] = (char) (unsigned char) i;
+        map[(int) (unsigned char) 'n'] = '\n';
+        map[(int) (unsigned char) 'r'] = '\r';
+        map[(int) (unsigned char) 't'] = '\t';
+    }
+    char operator[](char c) const { return map[(unsigned char) c]; }
+} escapes;
 }
 
 
@@ -213,10 +224,7 @@ static BackedStringView parseString(StringViewStream & str)
     for (c = content.begin(), end = content.end(); c != end; c++)
         if (*c == '\\') {
             c++;
-            if (*c == 'n') res += '\n';
-            else if (*c == 'r') res += '\r';
-            else if (*c == 't') res += '\t';
-            else res += *c;
+            res += escapes[*c];
         }
         else res += *c;
     return res;

From c62686a95bd3ebbf3f5104c27222e751e84b84a3 Mon Sep 17 00:00:00 2001
From: pennae <github@quasiparticle.net>
Date: Wed, 27 Dec 2023 04:26:50 +0100
Subject: [PATCH 297/421] reduce copies during drv parsing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

many paths need not be heap-allocated, and derivation env name/valye
pairs can be moved into the map.

before:

Benchmark 1: nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'
  Time (mean ± σ):      6.883 s ±  0.016 s    [User: 5.250 s, System: 1.424 s]
  Range (min … max):    6.860 s …  6.905 s    10 runs

after:

Benchmark 1: nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'
  Time (mean ± σ):      6.868 s ±  0.027 s    [User: 5.194 s, System: 1.466 s]
  Range (min … max):    6.828 s …  6.913 s    10 runs
---
 src/libstore/derivations.cc | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 89a345057..2fafcb8e7 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -235,10 +235,10 @@ static void validatePath(std::string_view s) {
         throw FormatError("bad path '%1%' in derivation", s);
 }
 
-static Path parsePath(StringViewStream & str)
+static BackedStringView parsePath(StringViewStream & str)
 {
-    auto s = parseString(str).toOwned();
-    validatePath(s);
+    auto s = parseString(str);
+    validatePath(*s);
     return s;
 }
 
@@ -262,7 +262,7 @@ static StringSet parseStrings(StringViewStream & str, bool arePaths)
     StringSet res;
     expect(str, "[");
     while (!endOfList(str))
-        res.insert(arePaths ? parsePath(str) : parseString(str).toOwned());
+        res.insert((arePaths ? parsePath(str) : parseString(str)).toOwned());
     return res;
 }
 
@@ -434,9 +434,9 @@ Derivation parseDerivation(
     expect(str, ",[");
     while (!endOfList(str)) {
         expect(str, "(");
-        Path drvPath = parsePath(str);
+        auto drvPath = parsePath(str);
         expect(str, ",");
-        drv.inputDrvs.map.insert_or_assign(store.parseStorePath(drvPath), parseDerivedPathMapNode(store, str, version));
+        drv.inputDrvs.map.insert_or_assign(store.parseStorePath(*drvPath), parseDerivedPathMapNode(store, str, version));
         expect(str, ")");
     }
 
@@ -455,7 +455,7 @@ Derivation parseDerivation(
         expect(str, "("); auto name = parseString(str).toOwned();
         expect(str, ","); auto value = parseString(str).toOwned();
         expect(str, ")");
-        drv.env[name] = value;
+        drv.env.insert_or_assign(std::move(name), std::move(value));
     }
 
     expect(str, ")");

From 1fe66852ff87e98615f35e8aac64675ff988fb5a Mon Sep 17 00:00:00 2001
From: pennae <github@quasiparticle.net>
Date: Fri, 22 Dec 2023 18:19:53 +0100
Subject: [PATCH 298/421] reduce the size of Env by one pointer
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

since `up` and `values` are both pointer-aligned the type field will
also be pointer-aligned, wasting 48 bits of space on most machines. we
can get away with removing the type field altogether by encoding some
information into the `with` expr that created the env to begin with,
reducing the GC load for the absolutely massive amount of single-entry
envs we create for lambdas. this reduces memory usage of system eval by
quite a bit (reducing heap size of our system eval from 8.4GB to 8.23GB)
and gives similar savings in eval time.

running `nix eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'`

before:

  Time (mean ± σ):      5.576 s ±  0.003 s    [User: 5.197 s, System: 0.378 s]
  Range (min … max):    5.572 s …  5.581 s    10 runs

after:

  Time (mean ± σ):      5.408 s ±  0.002 s    [User: 5.019 s, System: 0.388 s]
  Range (min … max):    5.405 s …  5.411 s    10 runs
---
 doc/manual/rl-next/env-size-reduction.md   |  7 +++++
 doc/manual/rl-next/with-error-reporting.md | 31 ++++++++++++++++++++++
 src/libcmd/repl.cc                         |  2 +-
 src/libexpr/eval-inline.hh                 |  2 --
 src/libexpr/eval.cc                        | 29 ++++++++++----------
 src/libexpr/eval.hh                        |  5 ----
 src/libexpr/nixexpr.cc                     | 18 ++++++++-----
 src/libexpr/nixexpr.hh                     | 13 ++++++---
 src/libexpr/primops.cc                     |  2 +-
 9 files changed, 75 insertions(+), 34 deletions(-)
 create mode 100644 doc/manual/rl-next/env-size-reduction.md
 create mode 100644 doc/manual/rl-next/with-error-reporting.md

diff --git a/doc/manual/rl-next/env-size-reduction.md b/doc/manual/rl-next/env-size-reduction.md
new file mode 100644
index 000000000..40a58bc28
--- /dev/null
+++ b/doc/manual/rl-next/env-size-reduction.md
@@ -0,0 +1,7 @@
+---
+synopsis: Reduce eval memory usage and wall time
+prs: 9658
+---
+
+Reduce the size of the `Env` struct used in the evaluator by a pointer, or 8 bytes on most modern machines.
+This reduces memory usage during eval by around 2% and wall time by around 3%.
diff --git a/doc/manual/rl-next/with-error-reporting.md b/doc/manual/rl-next/with-error-reporting.md
new file mode 100644
index 000000000..10b020956
--- /dev/null
+++ b/doc/manual/rl-next/with-error-reporting.md
@@ -0,0 +1,31 @@
+---
+synopsis: Better error reporting for `with` expressions
+prs: 9658
+---
+
+`with` expressions using non-attrset values to resolve variables are now reported with proper positions.
+
+Previously an incorrect `with` expression would report no position at all, making it hard to determine where the error originated:
+
+```
+nix-repl> with 1; a  
+error:
+       … <borked>
+
+         at «none»:0: (source not available)
+
+       error: value is an integer while a set was expected
+```
+
+Now position information is preserved and reported as with most other errors:
+
+```
+nix-repl> with 1; a
+error:
+       … while evaluating the first subexpression of a with expression
+         at «string»:1:1:
+            1| with 1; a
+             | ^
+
+       error: value is an integer while a set was expected
+```
diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc
index 97d709ff4..dea91ba63 100644
--- a/src/libcmd/repl.cc
+++ b/src/libcmd/repl.cc
@@ -112,7 +112,7 @@ NixRepl::NixRepl(const SearchPath & searchPath, nix::ref<Store> store, ref<EvalS
     : AbstractNixRepl(state)
     , debugTraceIndex(0)
     , getValues(getValues)
-    , staticEnv(new StaticEnv(false, state->staticBaseEnv.get()))
+    , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get()))
     , historyFile(getDataDir() + "/nix/repl-history")
 {
 }
diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index 52aa75b5f..f7710f819 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -73,8 +73,6 @@ Env & EvalState::allocEnv(size_t size)
 #endif
         env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
 
-    env->type = Env::Plain;
-
     /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */
 
     return *env;
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 810843995..ee1a87d9a 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -543,7 +543,7 @@ EvalState::EvalState(
     , env1AllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
 #endif
     , baseEnv(allocEnv(128))
-    , staticBaseEnv{std::make_shared<StaticEnv>(false, nullptr)}
+    , staticBaseEnv{std::make_shared<StaticEnv>(nullptr, nullptr)}
 {
     corepkgsFS->setPathDisplay("<nix", ">");
     internalFS->setPathDisplay("«nix-internal»", "");
@@ -781,7 +781,7 @@ void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se)
 // just for the current level of Env, not the whole chain.
 void printWithBindings(const SymbolTable & st, const Env & env)
 {
-    if (env.type == Env::HasWithAttrs) {
+    if (!env.values[0]->isThunk()) {
         std::cout << "with: ";
         std::cout << ANSI_MAGENTA;
         Bindings::iterator j = env.values[0]->attrs->begin();
@@ -835,7 +835,7 @@ void mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const En
     if (env.up && se.up) {
         mapStaticEnvBindings(st, *se.up, *env.up, vm);
 
-        if (env.type == Env::HasWithAttrs) {
+        if (!env.values[0]->isThunk()) {
             // add 'with' bindings.
             Bindings::iterator j = env.values[0]->attrs->begin();
             while (j != env.values[0]->attrs->end()) {
@@ -973,22 +973,23 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
 
     if (!var.fromWith) return env->values[var.displ];
 
+    // This early exit defeats the `maybeThunk` optimization for variables from `with`,
+    // The added complexity of handling this appears to be similarly in cost, or
+    // the cases where applicable were insignificant in the first place.
+    if (noEval) return nullptr;
+
+    auto * fromWith = var.fromWith;
     while (1) {
-        if (env->type == Env::HasWithExpr) {
-            if (noEval) return 0;
-            Value * v = allocValue();
-            evalAttrs(*env->up, (Expr *) env->values[0], *v, noPos, "<borked>");
-            env->values[0] = v;
-            env->type = Env::HasWithAttrs;
-        }
+        forceAttrs(*env->values[0], fromWith->pos, "while evaluating the first subexpression of a with expression");
         Bindings::iterator j = env->values[0]->attrs->find(var.name);
         if (j != env->values[0]->attrs->end()) {
             if (countCalls) attrSelects[j->pos]++;
             return j->value;
         }
-        if (!env->prevWith)
+        if (!fromWith->parentWith)
             error("undefined variable '%1%'", symbols[var.name]).atPos(var.pos).withFrame(*env, var).debugThrow<UndefinedVarError>();
-        for (size_t l = env->prevWith; l; --l, env = env->up) ;
+        for (size_t l = fromWith->prevWith; l; --l, env = env->up) ;
+        fromWith = fromWith->parentWith;
     }
 }
 
@@ -1816,9 +1817,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v)
 {
     Env & env2(state.allocEnv(1));
     env2.up = &env;
-    env2.prevWith = prevWith;
-    env2.type = Env::HasWithExpr;
-    env2.values[0] = (Value *) attrs;
+    env2.values[0] = attrs->maybeThunk(state, env);
 
     body->eval(state, env2, v);
 }
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index da2d256db..db606ebae 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -116,11 +116,6 @@ struct Constant
 struct Env
 {
     Env * up;
-    /**
-     * Number of of levels up to next `with` environment
-     */
-    unsigned short prevWith:14;
-    enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2;
     Value * values[0];
 };
 
diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc
index 84860b30f..ede070cff 100644
--- a/src/libexpr/nixexpr.cc
+++ b/src/libexpr/nixexpr.cc
@@ -333,6 +333,8 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
+    fromWith = nullptr;
+
     /* Check whether the variable appears in the environment.  If so,
        set its level and displacement. */
     const StaticEnv * curEnv;
@@ -344,7 +346,6 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
         } else {
             auto i = curEnv->find(name);
             if (i != curEnv->vars.end()) {
-                fromWith = false;
                 this->level = level;
                 displ = i->second;
                 return;
@@ -360,7 +361,8 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
             .msg = hintfmt("undefined variable '%1%'", es.symbols[name]),
             .errPos = es.positions[pos]
         });
-    fromWith = true;
+    for (auto * e = env.get(); e && !fromWith; e = e->up)
+        fromWith = e->isWith;
     this->level = withLevel;
 }
 
@@ -393,7 +395,7 @@ void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
         es.exprEnvs.insert(std::make_pair(this, env));
 
     if (recursive) {
-        auto newEnv = std::make_shared<StaticEnv>(false, env.get(), recursive ? attrs.size() : 0);
+        auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), recursive ? attrs.size() : 0);
 
         Displacement displ = 0;
         for (auto & i : attrs)
@@ -435,7 +437,7 @@ void ExprLambda::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
         es.exprEnvs.insert(std::make_pair(this, env));
 
     auto newEnv = std::make_shared<StaticEnv>(
-        false, env.get(),
+        nullptr, env.get(),
         (hasFormals() ? formals->formals.size() : 0) +
         (!arg ? 0 : 1));
 
@@ -471,7 +473,7 @@ void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
-    auto newEnv = std::make_shared<StaticEnv>(false, env.get(), attrs->attrs.size());
+    auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), attrs->attrs.size());
 
     Displacement displ = 0;
     for (auto & i : attrs->attrs)
@@ -490,6 +492,10 @@ void ExprWith::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
+    parentWith = nullptr;
+    for (auto * e = env.get(); e && !parentWith; e = e->up)
+        parentWith = e->isWith;
+
     /* Does this `with' have an enclosing `with'?  If so, record its
        level so that `lookupVar' can look up variables in the previous
        `with' if this one doesn't contain the desired attribute. */
@@ -506,7 +512,7 @@ void ExprWith::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
         es.exprEnvs.insert(std::make_pair(this, env));
 
     attrs->bindVars(es, env);
-    auto newEnv = std::make_shared<StaticEnv>(true, env.get());
+    auto newEnv = std::make_shared<StaticEnv>(this, env.get());
     body->bindVars(es, newEnv);
 }
 
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index 1e57fec7a..e50a157ee 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -138,6 +138,7 @@ std::ostream & operator << (std::ostream & str, const Pos & pos);
 struct Env;
 struct Value;
 class EvalState;
+struct ExprWith;
 struct StaticEnv;
 
 
@@ -226,8 +227,11 @@ struct ExprVar : Expr
     Symbol name;
 
     /* Whether the variable comes from an environment (e.g. a rec, let
-       or function argument) or from a "with". */
-    bool fromWith;
+       or function argument) or from a "with".
+
+       `nullptr`: Not from a `with`.
+       Valid pointer: the nearest, innermost `with` expression to query first. */
+    ExprWith * fromWith;
 
     /* In the former case, the value is obtained by going `level`
        levels up from the current environment and getting the
@@ -385,6 +389,7 @@ struct ExprWith : Expr
     PosIdx pos;
     Expr * attrs, * body;
     size_t prevWith;
+    ExprWith * parentWith;
     ExprWith(const PosIdx & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { };
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
@@ -478,14 +483,14 @@ extern ExprBlackHole eBlackHole;
    runtime. */
 struct StaticEnv
 {
-    bool isWith;
+    ExprWith * isWith;
     const StaticEnv * up;
 
     // Note: these must be in sorted order.
     typedef std::vector<std::pair<Symbol, Displacement>> Vars;
     Vars vars;
 
-    StaticEnv(bool isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) {
+    StaticEnv(ExprWith * isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) {
         vars.reserve(expectedSize);
     };
 
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index a1502da45..924de3184 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -214,7 +214,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
             Env * env = &state.allocEnv(vScope->attrs->size());
             env->up = &state.baseEnv;
 
-            auto staticEnv = std::make_shared<StaticEnv>(false, state.staticBaseEnv.get(), vScope->attrs->size());
+            auto staticEnv = std::make_shared<StaticEnv>(nullptr, state.staticBaseEnv.get(), vScope->attrs->size());
 
             unsigned int displ = 0;
             for (auto & attr : *vScope->attrs) {

From 3f796514b37a1e723a395fce8271428410e93f5f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 2 Jan 2024 12:39:16 +0100
Subject: [PATCH 299/421] Optimize empty list constants
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This avoids a Value allocation for empty list constants. During a `nix
search nixpkgs`, about 82% of all thunked lists are empty, so this
removes about 3 million Value allocations.

Performance comparison on `nix search github:NixOS/nixpkgs/e1fa12d4f6c6fe19ccb59cac54b5b3f25e160870 --no-eval-cache`:

maximum RSS:        median = 3845432.0000  mean = 3845432.0000  stddev =      0.0000  min = 3845432.0000  max = 3845432.0000  [rejected?, p=0.00000, Δ=-70084.00000±0.00000]
soft page faults:   median = 965395.0000  mean = 965394.6667  stddev =      1.1181  min = 965392.0000  max = 965396.0000  [rejected?, p=0.00000, Δ=-17929.77778±38.59610]
system CPU time:    median =      1.8029  mean =      1.7702  stddev =      0.0621  min =      1.6749  max =      1.8417  [rejected, p=0.00064, Δ=-0.12873±0.09905]
user CPU time:      median =     14.1022  mean =     14.0633  stddev =      0.1869  min =     13.8118  max =     14.3190  [not rejected, p=0.03006, Δ=-0.18248±0.24928]
elapsed time:       median =     15.8205  mean =     15.8618  stddev =      0.2312  min =     15.5033  max =     16.1670  [not rejected, p=0.00558, Δ=-0.28963±0.29434]
---
 src/libexpr/eval.cc    | 11 +++++++++++
 src/libexpr/eval.hh    |  3 +++
 src/libexpr/nixexpr.hh |  1 +
 3 files changed, 15 insertions(+)

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 810843995..494b8338f 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -554,6 +554,8 @@ EvalState::EvalState(
 
     static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes");
 
+    vEmptyList.mkList(0);
+
     /* Initialise the Nix expression search path. */
     if (!evalSettings.pureEval) {
         for (auto & i : _searchPath.elements)
@@ -1384,6 +1386,15 @@ void ExprList::eval(EvalState & state, Env & env, Value & v)
 }
 
 
+Value * ExprList::maybeThunk(EvalState & state, Env & env)
+{
+    if (elems.empty()) {
+        return &state.vEmptyList;
+    }
+    return Expr::maybeThunk(state, env);
+}
+
+
 void ExprVar::eval(EvalState & state, Env & env, Value & v)
 {
     Value * v2 = state.lookupVar(&env, *this, false);
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index da2d256db..bf85b50c8 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -305,6 +305,9 @@ public:
         return *errorBuilder;
     }
 
+    /* Empty list constant. */
+    Value vEmptyList;
+
 private:
 
     /* Cache for calls to addToStore(); maps source paths to the store
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index 1e57fec7a..55e930758 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -299,6 +299,7 @@ struct ExprList : Expr
     std::vector<Expr *> elems;
     ExprList() { };
     COMMON_METHODS
+    Value * maybeThunk(EvalState & state, Env & env) override;
 
     PosIdx getPos() const override
     {

From 2b20f36f9515882589975d14a94ba1fd2b5c513a Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 2 Jan 2024 12:33:51 -0500
Subject: [PATCH 300/421] Fix NetBSD build

There was still a mistake after my earlier
a7115a47ef0d83ea81b494f6bc5b11d8286e0672 and
e13fc0bbdb1e1eefeb33ff4d18310958041b1ad5. This finally gets it right.
---
 configure.ac                          | 7 ++++++-
 src/libstore/globals.hh               | 2 ++
 src/libstore/posix-fs-canonicalise.cc | 4 ++--
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/configure.ac b/configure.ac
index 1bc4f17b0..b97e25bbd 100644
--- a/configure.ac
+++ b/configure.ac
@@ -308,7 +308,12 @@ AC_SUBST(HAVE_SECCOMP, [$have_seccomp])
 
 # Optional dependencies for better normalizing file system data
 AC_CHECK_HEADERS([sys/xattr.h])
-AC_CHECK_FUNCS([llistxattr lremovexattr])
+AS_IF([test "$ac_cv_header_sys_xattr_h" = "yes"],[
+  AC_CHECK_FUNCS([llistxattr lremovexattr])
+  AS_IF([test "$ac_cv_func_llistxattr" = "yes" && test "$ac_cv_func_lremovexattr" = "yes"],[
+    AC_DEFINE([HAVE_ACL_SUPPORT], [1], [Define if we can manipulate file system Access Control Lists])
+  ])
+])
 
 # Look for aws-cpp-sdk-s3.
 AC_LANG_PUSH(C++)
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index b35dc37a1..cf34ae354 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -946,7 +946,9 @@ public:
           may be useful in certain scenarios (e.g. to spin up containers or
           set up userspace network interfaces in tests).
         )"};
+#endif
 
+#if HAVE_ACL_SUPPORT
     Setting<StringSet> ignoredAcls{
         this, {"security.selinux", "system.nfs4_acl", "security.csm"}, "ignored-acls",
         R"(
diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc
index 5edda0157..8b29e90d4 100644
--- a/src/libstore/posix-fs-canonicalise.cc
+++ b/src/libstore/posix-fs-canonicalise.cc
@@ -1,4 +1,4 @@
-#if HAVE_SYS_XATTR_H
+#if HAVE_ACL_SUPPORT
 # include <sys/xattr.h>
 #endif
 
@@ -78,7 +78,7 @@ static void canonicalisePathMetaData_(
     if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
         throw Error("file '%1%' has an unsupported type", path);
 
-#if HAVE_SYS_XATTR_H && HAVE_LLISTXATTR && HAVE_LREMOVEXATTR
+#if HAVE_ACL_SUPPORT
     /* Remove extended attributes / ACLs. */
     ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0);
 

From 86e924443722a04f7d458594e3332ffaa73edb1d Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 2 Jan 2024 12:41:53 -0500
Subject: [PATCH 301/421] Fix `buildNoTest`

`checkInputs` is not right for this because we don't just need these
deps when `doTest`, we also need them when `installUnitTests`.
---
 package.nix | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/package.nix b/package.nix
index b5ff45083..56276ecc4 100644
--- a/package.nix
+++ b/package.nix
@@ -214,6 +214,9 @@ in {
   ] ++ lib.optionals (!stdenv.hostPlatform.isWindows) [
     editline
     lowdown
+  ] ++ lib.optionals buildUnitTests [
+    gtest
+    rapidcheck
   ] ++ lib.optional stdenv.isLinux libseccomp
     ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid
     # There have been issues building these dependencies
@@ -232,11 +235,6 @@ in {
   dontBuild = !attrs.doBuild;
   doCheck = attrs.doCheck;
 
-  checkInputs = [
-    gtest
-    rapidcheck
-  ];
-
   nativeCheckInputs = [
     git
     mercurial

From 7b8af5f916a73aa5927b103ff712280023cea840 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 2 Jan 2024 12:50:48 -0500
Subject: [PATCH 302/421] `buildNoTests`: Restore intent

The thing we wanted to test was that building Nix without building or
running tests, and without depending on libraries only needed by tests,
works.

But since 6c8f4ef3502aa214557541ec00538e41aeced6e3, we can also install
unit tests, and during the conversion to using `package.nix` this
started happening more often (they go to a separate output though, so
this should be fine).

This adds more `... = false` to restore the original intent: don't run
unit test or functional tests, and don't install unit tests.
---
 flake.nix | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/flake.nix b/flake.nix
index a8fc105e8..9217de9af 100644
--- a/flake.nix
+++ b/flake.nix
@@ -234,11 +234,11 @@
         buildNoGc = forAllSystems (system: self.packages.${system}.nix.overrideAttrs (a: { configureFlags = (a.configureFlags or []) ++ ["--enable-gc=no"];}));
 
         buildNoTests = forAllSystems (system:
-          self.packages.${system}.nix.overrideAttrs (a: {
-            doCheck =
-              assert ! a?dontCheck;
-              false;
-          })
+          self.packages.${system}.nix.override {
+            doCheck = false;
+            doInstallCheck = false;
+            installUnitTests = false;
+          }
         );
 
         # Perl bindings for various platforms.

From 484881f3021856cd0d0c0cb42d4473b3c7ea0051 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 3 Jan 2024 10:23:27 +0100
Subject: [PATCH 303/421] Move empty list constant

---
 src/libexpr/eval.hh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index bf85b50c8..e2180f00d 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -218,6 +218,11 @@ public:
 
     Bindings emptyBindings;
 
+    /**
+     * Empty list constant.
+     */
+    Value vEmptyList;
+
     /**
      * The accessor for the root filesystem.
      */
@@ -305,9 +310,6 @@ public:
         return *errorBuilder;
     }
 
-    /* Empty list constant. */
-    Value vEmptyList;
-
 private:
 
     /* Cache for calls to addToStore(); maps source paths to the store

From 24e70489e59f9ab75310382dc59df09796ea8df4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 3 Jan 2024 19:14:50 +0100
Subject: [PATCH 304/421] withFramedSink(): Receive interrupts on the stderr
 thread

Otherwise Nix deadlocks when Ctrl-C is received in withFramedSink():
the parent thread will wait forever for the stderr thread to shut
down.

Fixes the hang reported in https://github.com/NixOS/nix/issues/7245#issuecomment-1770560923.
---
 src/libstore/remote-store.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index f0df646ca..078b9fe00 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -16,6 +16,8 @@
 #include "logging.hh"
 #include "callback.hh"
 #include "filetransfer.hh"
+#include "signals.hh"
+
 #include <nlohmann/json.hpp>
 
 namespace nix {
@@ -1066,6 +1068,7 @@ void RemoteStore::ConnectionHandle::withFramedSink(std::function<void(Sink & sin
     std::thread stderrThread([&]()
     {
         try {
+            ReceiveInterrupts receiveInterrupts;
             processStderr(nullptr, nullptr, false);
         } catch (...) {
             ex = std::current_exception();

From 295a2ff8bdff0f681e0c5c54e9378460eb63b663 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 3 Jan 2024 19:30:02 +0100
Subject: [PATCH 305/421] Make some more threads receive interrupts

Shouldn't hurt to do this. In particular, this should speed up
shutting down the PathSubstitutionGoal thread if it's copying from a
remote store.
---
 src/libstore/build/substitution-goal.cc | 3 +++
 src/libutil/thread-pool.cc              | 2 ++
 2 files changed, 5 insertions(+)

diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc
index 93867007d..c7e8e2825 100644
--- a/src/libstore/build/substitution-goal.cc
+++ b/src/libstore/build/substitution-goal.cc
@@ -2,6 +2,7 @@
 #include "substitution-goal.hh"
 #include "nar-info.hh"
 #include "finally.hh"
+#include "signals.hh"
 
 namespace nix {
 
@@ -217,6 +218,8 @@ void PathSubstitutionGoal::tryToRun()
 
     thr = std::thread([this]() {
         try {
+            ReceiveInterrupts receiveInterrupts;
+
             /* Wake up the worker loop when we're done. */
             Finally updateStats([this]() { outPipe.writeSide.close(); });
 
diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc
index c5e735617..9a7dfee56 100644
--- a/src/libutil/thread-pool.cc
+++ b/src/libutil/thread-pool.cc
@@ -79,6 +79,8 @@ void ThreadPool::process()
 
 void ThreadPool::doWork(bool mainThread)
 {
+    ReceiveInterrupts receiveInterrupts;
+
     if (!mainThread)
         interruptCheck = [&]() { return (bool) quit; };
 

From 12bb8cdd381156456a712e4a5a8af3b6bc852eab Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 3 Jan 2024 15:02:20 -0500
Subject: [PATCH 306/421] Signer infrastructure: Prep for #9076

This sets up infrastructure in libutil to allow for signing other than
by a secret key in memory. #9076 uses this to implement remote signing.

(Split from that PR to allow reviewing in smaller chunks.)

Co-Authored-By: Raito Bezarius <masterancpp@gmail.com>
---
 perl/lib/Nix/Store.xs                         |  1 -
 src/libstore/binary-cache-store.cc            |  5 +-
 src/libstore/binary-cache-store.hh            |  5 +-
 src/libstore/globals.cc                       |  5 --
 src/libstore/keys.cc                          | 31 ++++++++++
 src/libstore/keys.hh                          | 10 +++
 src/libstore/local-store.cc                   |  7 ++-
 src/libstore/local.mk                         |  2 +-
 src/libstore/path-info.cc                     |  4 +-
 src/libstore/path-info.hh                     |  4 +-
 src/libstore/path.cc                          |  6 +-
 src/libstore/realisation.cc                   |  5 +-
 src/libstore/realisation.hh                   |  4 +-
 src/libstore/store-api.cc                     |  2 +-
 src/libutil/hash.cc                           |  9 +++
 src/libutil/hash.hh                           |  6 +-
 src/libutil/local.mk                          |  7 ++-
 .../signature/local-keys.cc}                  | 54 +++++++---------
 .../signature/local-keys.hh}                  | 42 +++++++++++--
 src/libutil/signature/signer.cc               | 23 +++++++
 src/libutil/signature/signer.hh               | 61 +++++++++++++++++++
 src/libutil/util.cc                           |  4 ++
 src/nix/sigs.cc                               |  5 +-
 src/nix/verify.cc                             |  1 +
 24 files changed, 233 insertions(+), 70 deletions(-)
 create mode 100644 src/libstore/keys.cc
 create mode 100644 src/libstore/keys.hh
 rename src/{libstore/crypto.cc => libutil/signature/local-keys.cc} (64%)
 rename src/{libstore/crypto.hh => libutil/signature/local-keys.hh} (51%)
 create mode 100644 src/libutil/signature/signer.cc
 create mode 100644 src/libutil/signature/signer.hh

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index 4964b8a34..423c01cf7 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -12,7 +12,6 @@
 #include "realisation.hh"
 #include "globals.hh"
 #include "store-api.hh"
-#include "crypto.hh"
 #include "posix-source-accessor.hh"
 
 #include <sodium.h>
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 8a3052433..ea1279e2e 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -28,7 +28,8 @@ BinaryCacheStore::BinaryCacheStore(const Params & params)
     , Store(params)
 {
     if (secretKeyFile != "")
-        secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
+        signer = std::make_unique<LocalSigner>(
+            SecretKey { readFile(secretKeyFile) });
 
     StringSink sink;
     sink << narVersionMagic1;
@@ -274,7 +275,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     stats.narWriteCompressionTimeMs += duration;
 
     /* Atomically write the NAR info file.*/
-    if (secretKey) narInfo->sign(*this, *secretKey);
+    if (signer) narInfo->sign(*this, *signer);
 
     writeNarInfo(narInfo);
 
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 98e43ee6a..00ab73905 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -1,7 +1,7 @@
 #pragma once
 ///@file
 
-#include "crypto.hh"
+#include "signature/local-keys.hh"
 #include "store-api.hh"
 #include "log-store.hh"
 
@@ -57,8 +57,7 @@ class BinaryCacheStore : public virtual BinaryCacheStoreConfig,
 {
 
 private:
-
-    std::unique_ptr<SecretKey> secretKey;
+    std::unique_ptr<Signer> signer;
 
 protected:
 
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index f401d076d..50584e06c 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -15,8 +15,6 @@
 
 #include <nlohmann/json.hpp>
 
-#include <sodium/core.h>
-
 #ifdef __GLIBC__
 # include <gnu/lib-names.h>
 # include <nss.h>
@@ -409,9 +407,6 @@ void initLibStore() {
 
     initLibUtil();
 
-    if (sodium_init() == -1)
-        throw Error("could not initialise libsodium");
-
     loadConfFile();
 
     preloadNSS();
diff --git a/src/libstore/keys.cc b/src/libstore/keys.cc
new file mode 100644
index 000000000..2cc50970f
--- /dev/null
+++ b/src/libstore/keys.cc
@@ -0,0 +1,31 @@
+#include "file-system.hh"
+#include "globals.hh"
+#include "keys.hh"
+
+namespace nix {
+
+PublicKeys getDefaultPublicKeys()
+{
+    PublicKeys publicKeys;
+
+    // FIXME: filter duplicates
+
+    for (auto s : settings.trustedPublicKeys.get()) {
+        PublicKey key(s);
+        publicKeys.emplace(key.name, key);
+    }
+
+    for (auto secretKeyFile : settings.secretKeyFiles.get()) {
+        try {
+            SecretKey secretKey(readFile(secretKeyFile));
+            publicKeys.emplace(secretKey.name, secretKey.toPublicKey());
+        } catch (SysError & e) {
+            /* Ignore unreadable key files. That's normal in a
+               multi-user installation. */
+        }
+    }
+
+    return publicKeys;
+}
+
+}
diff --git a/src/libstore/keys.hh b/src/libstore/keys.hh
new file mode 100644
index 000000000..3da19493f
--- /dev/null
+++ b/src/libstore/keys.hh
@@ -0,0 +1,10 @@
+#pragma once
+///@file
+
+#include "signature/local-keys.hh"
+
+namespace nix {
+
+PublicKeys getDefaultPublicKeys();
+
+}
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 63e90ea1e..0f3c37c8a 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -14,6 +14,7 @@
 #include "signals.hh"
 #include "posix-fs-canonicalise.hh"
 #include "posix-source-accessor.hh"
+#include "keys.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -1578,7 +1579,8 @@ void LocalStore::signRealisation(Realisation & realisation)
 
     for (auto & secretKeyFile : secretKeyFiles.get()) {
         SecretKey secretKey(readFile(secretKeyFile));
-        realisation.sign(secretKey);
+        LocalSigner signer(std::move(secretKey));
+        realisation.sign(signer);
     }
 }
 
@@ -1590,7 +1592,8 @@ void LocalStore::signPathInfo(ValidPathInfo & info)
 
     for (auto & secretKeyFile : secretKeyFiles.get()) {
         SecretKey secretKey(readFile(secretKeyFile));
-        info.sign(*this, secretKey);
+        LocalSigner signer(std::move(secretKey));
+        info.sign(*this, signer);
     }
 }
 
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index 68ccdc409..675976314 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -8,7 +8,7 @@ libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc)
 
 libstore_LIBS = libutil
 
-libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
+libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) -pthread
 ifdef HOST_LINUX
  libstore_LDFLAGS += -ldl
 endif
diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc
index f58e31bfd..d82ccd0c9 100644
--- a/src/libstore/path-info.cc
+++ b/src/libstore/path-info.cc
@@ -38,9 +38,9 @@ std::string ValidPathInfo::fingerprint(const Store & store) const
 }
 
 
-void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
+void ValidPathInfo::sign(const Store & store, const Signer & signer)
 {
-    sigs.insert(secretKey.signDetached(fingerprint(store)));
+    sigs.insert(signer.signDetached(fingerprint(store)));
 }
 
 std::optional<ContentAddressWithReferences> ValidPathInfo::contentAddressWithReferences() const
diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh
index 077abc7e1..b6dc0855d 100644
--- a/src/libstore/path-info.hh
+++ b/src/libstore/path-info.hh
@@ -1,7 +1,7 @@
 #pragma once
 ///@file
 
-#include "crypto.hh"
+#include "signature/signer.hh"
 #include "path.hh"
 #include "hash.hh"
 #include "content-address.hh"
@@ -107,7 +107,7 @@ struct ValidPathInfo : UnkeyedValidPathInfo {
      */
     std::string fingerprint(const Store & store) const;
 
-    void sign(const Store & store, const SecretKey & secretKey);
+    void sign(const Store & store, const Signer & signer);
 
     /**
      * @return The `ContentAddressWithReferences` that determines the
diff --git a/src/libstore/path.cc b/src/libstore/path.cc
index 1afd10af7..a15a78545 100644
--- a/src/libstore/path.cc
+++ b/src/libstore/path.cc
@@ -1,7 +1,5 @@
 #include "store-dir-config.hh"
 
-#include <sodium.h>
-
 namespace nix {
 
 static void checkName(std::string_view path, std::string_view name)
@@ -49,9 +47,7 @@ StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x");
 
 StorePath StorePath::random(std::string_view name)
 {
-    Hash hash(HashAlgorithm::SHA1);
-    randombytes_buf(hash.hash, hash.hashSize);
-    return StorePath(hash, name);
+    return StorePath(Hash::random(HashAlgorithm::SHA1), name);
 }
 
 StorePath StoreDirConfig::parseStorePath(std::string_view path) const
diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc
index 93ddb5b20..86bfdd1a8 100644
--- a/src/libstore/realisation.cc
+++ b/src/libstore/realisation.cc
@@ -1,6 +1,7 @@
 #include "realisation.hh"
 #include "store-api.hh"
 #include "closure.hh"
+#include "signature/local-keys.hh"
 #include <nlohmann/json.hpp>
 
 namespace nix {
@@ -113,9 +114,9 @@ std::string Realisation::fingerprint() const
     return serialized.dump();
 }
 
-void Realisation::sign(const SecretKey & secretKey)
+void Realisation::sign(const Signer &signer)
 {
-    signatures.insert(secretKey.signDetached(fingerprint()));
+    signatures.insert(signer.signDetached(fingerprint()));
 }
 
 bool Realisation::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const
diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh
index 4ba2123d8..ddb4af770 100644
--- a/src/libstore/realisation.hh
+++ b/src/libstore/realisation.hh
@@ -8,7 +8,7 @@
 #include "derived-path.hh"
 #include <nlohmann/json_fwd.hpp>
 #include "comparator.hh"
-#include "crypto.hh"
+#include "signature/signer.hh"
 
 namespace nix {
 
@@ -64,7 +64,7 @@ struct Realisation {
     static Realisation fromJSON(const nlohmann::json& json, const std::string& whence);
 
     std::string fingerprint() const;
-    void sign(const SecretKey &);
+    void sign(const Signer &);
     bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
     size_t checkSignatures(const PublicKeys & publicKeys) const;
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index c2516afb5..c48bfc248 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -1,4 +1,4 @@
-#include "crypto.hh"
+#include "signature/local-keys.hh"
 #include "source-accessor.hh"
 #include "globals.hh"
 #include "derived-path.hh"
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 502afbda2..d067da969 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -14,6 +14,8 @@
 #include <sys/stat.h>
 #include <fcntl.h>
 
+#include <sodium.h>
+
 namespace nix {
 
 static size_t regularHashSize(HashAlgorithm type) {
@@ -261,6 +263,13 @@ Hash::Hash(std::string_view rest, HashAlgorithm algo, bool isSRI)
         throw BadHash("hash '%s' has wrong length for hash algorithm '%s'", rest, printHashAlgo(this->algo));
 }
 
+Hash Hash::random(HashAlgorithm algo)
+{
+    Hash hash(algo);
+    randombytes_buf(hash.hash, hash.hashSize);
+    return hash;
+}
+
 Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashAlgorithm> ha)
 {
     if (hashStr.empty()) {
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index 2fe9a53f5..f7e8eb265 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -5,7 +5,6 @@
 #include "serialise.hh"
 #include "file-system.hh"
 
-
 namespace nix {
 
 
@@ -143,6 +142,11 @@ public:
     }
 
     static Hash dummy;
+
+    /**
+     * @return a random hash with hash algorithm `algo`
+     */
+    static Hash random(HashAlgorithm algo);
 };
 
 /**
diff --git a/src/libutil/local.mk b/src/libutil/local.mk
index 81efaafec..0fdebaf5c 100644
--- a/src/libutil/local.mk
+++ b/src/libutil/local.mk
@@ -4,14 +4,17 @@ libutil_NAME = libnixutil
 
 libutil_DIR := $(d)
 
-libutil_SOURCES := $(wildcard $(d)/*.cc)
+libutil_SOURCES := $(wildcard $(d)/*.cc $(d)/signature/*.cc)
 
 libutil_CXXFLAGS += -I src/libutil
 
-libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
+libutil_LDFLAGS += -pthread $(LIBCURL_LIBS) $(SODIUM_LIBS) $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
 
 $(foreach i, $(wildcard $(d)/args/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/args, 0644)))
+$(foreach i, $(wildcard $(d)/signature/*.hh), \
+  $(eval $(call install-file-in, $(i), $(includedir)/nix/signature, 0644)))
+
 
 ifeq ($(HAVE_LIBCPUID), 1)
 	libutil_LDFLAGS += -lcpuid
diff --git a/src/libstore/crypto.cc b/src/libutil/signature/local-keys.cc
similarity index 64%
rename from src/libstore/crypto.cc
rename to src/libutil/signature/local-keys.cc
index 1b705733c..858b036f5 100644
--- a/src/libstore/crypto.cc
+++ b/src/libutil/signature/local-keys.cc
@@ -1,13 +1,12 @@
-#include "crypto.hh"
+#include "signature/local-keys.hh"
+
 #include "file-system.hh"
 #include "util.hh"
-#include "globals.hh"
-
 #include <sodium.h>
 
 namespace nix {
 
-static std::pair<std::string_view, std::string_view> split(std::string_view s)
+BorrowedCryptoValue BorrowedCryptoValue::parse(std::string_view s)
 {
     size_t colon = s.find(':');
     if (colon == std::string::npos || colon == 0)
@@ -17,10 +16,10 @@ static std::pair<std::string_view, std::string_view> split(std::string_view s)
 
 Key::Key(std::string_view s)
 {
-    auto ss = split(s);
+    auto ss = BorrowedCryptoValue::parse(s);
 
-    name = ss.first;
-    key = ss.second;
+    name = ss.name;
+    key = ss.payload;
 
     if (name == "" || key == "")
         throw Error("secret key is corrupt");
@@ -73,45 +72,34 @@ PublicKey::PublicKey(std::string_view s)
         throw Error("public key is not valid");
 }
 
-bool verifyDetached(const std::string & data, const std::string & sig,
-    const PublicKeys & publicKeys)
+bool PublicKey::verifyDetached(std::string_view data, std::string_view sig) const
 {
-    auto ss = split(sig);
+    auto ss = BorrowedCryptoValue::parse(sig);
 
-    auto key = publicKeys.find(std::string(ss.first));
-    if (key == publicKeys.end()) return false;
+    if (ss.name != std::string_view { name }) return false;
 
-    auto sig2 = base64Decode(ss.second);
+    return verifyDetachedAnon(data, ss.payload);
+}
+
+bool PublicKey::verifyDetachedAnon(std::string_view data, std::string_view sig) const
+{
+    auto sig2 = base64Decode(sig);
     if (sig2.size() != crypto_sign_BYTES)
         throw Error("signature is not valid");
 
     return crypto_sign_verify_detached((unsigned char *) sig2.data(),
         (unsigned char *) data.data(), data.size(),
-        (unsigned char *) key->second.key.data()) == 0;
+        (unsigned char *) key.data()) == 0;
 }
 
-PublicKeys getDefaultPublicKeys()
+bool verifyDetached(std::string_view data, std::string_view sig, const PublicKeys & publicKeys)
 {
-    PublicKeys publicKeys;
+    auto ss = BorrowedCryptoValue::parse(sig);
 
-    // FIXME: filter duplicates
+    auto key = publicKeys.find(std::string(ss.name));
+    if (key == publicKeys.end()) return false;
 
-    for (auto s : settings.trustedPublicKeys.get()) {
-        PublicKey key(s);
-        publicKeys.emplace(key.name, key);
-    }
-
-    for (auto secretKeyFile : settings.secretKeyFiles.get()) {
-        try {
-            SecretKey secretKey(readFile(secretKeyFile));
-            publicKeys.emplace(secretKey.name, secretKey.toPublicKey());
-        } catch (SysError & e) {
-            /* Ignore unreadable key files. That's normal in a
-               multi-user installation. */
-        }
-    }
-
-    return publicKeys;
+    return key->second.verifyDetachedAnon(data, ss.payload);
 }
 
 }
diff --git a/src/libstore/crypto.hh b/src/libutil/signature/local-keys.hh
similarity index 51%
rename from src/libstore/crypto.hh
rename to src/libutil/signature/local-keys.hh
index 35216d470..4aafc1239 100644
--- a/src/libstore/crypto.hh
+++ b/src/libutil/signature/local-keys.hh
@@ -7,6 +7,25 @@
 
 namespace nix {
 
+/**
+ * Except where otherwise noted, Nix serializes keys and signatures in
+ * the form:
+ *
+ * ```
+ * <name>:<key/signature-in-Base64>
+ * ```
+ */
+struct BorrowedCryptoValue {
+    std::string_view name;
+    std::string_view payload;
+
+    /**
+     * This splits on the colon, the user can then separated decode the
+     * Base64 payload separately.
+     */
+    static BorrowedCryptoValue parse(std::string_view);
+};
+
 struct Key
 {
     std::string name;
@@ -49,21 +68,36 @@ struct PublicKey : Key
 {
     PublicKey(std::string_view data);
 
+    /**
+     * @return true iff `sig` and this key's names match, and `sig` is a
+     * correct signature over `data` using the given public key.
+     */
+    bool verifyDetached(std::string_view data, std::string_view sigs) const;
+
+    /**
+     * @return true iff `sig` is a correct signature over `data` using the
+     * given public key.
+     *
+     * @param just the Base64 signature itself, not a colon-separated pair of a
+     * public key name and signature.
+     */
+    bool verifyDetachedAnon(std::string_view data, std::string_view sigs) const;
+
 private:
     PublicKey(std::string_view name, std::string && key)
         : Key(name, std::move(key)) { }
     friend struct SecretKey;
 };
 
+/**
+ * Map from key names to public keys
+ */
 typedef std::map<std::string, PublicKey> PublicKeys;
 
 /**
  * @return true iff ‘sig’ is a correct signature over ‘data’ using one
  * of the given public keys.
  */
-bool verifyDetached(const std::string & data, const std::string & sig,
-    const PublicKeys & publicKeys);
-
-PublicKeys getDefaultPublicKeys();
+bool verifyDetached(std::string_view data, std::string_view sig, const PublicKeys & publicKeys);
 
 }
diff --git a/src/libutil/signature/signer.cc b/src/libutil/signature/signer.cc
new file mode 100644
index 000000000..0d26867b5
--- /dev/null
+++ b/src/libutil/signature/signer.cc
@@ -0,0 +1,23 @@
+#include "signature/signer.hh"
+#include "error.hh"
+
+#include <sodium.h>
+
+namespace nix {
+
+LocalSigner::LocalSigner(SecretKey && privateKey)
+    : privateKey(privateKey)
+    , publicKey(privateKey.toPublicKey())
+{ }
+
+std::string LocalSigner::signDetached(std::string_view s) const
+{
+    return privateKey.signDetached(s);
+}
+
+const PublicKey & LocalSigner::getPublicKey()
+{
+    return publicKey;
+}
+
+}
diff --git a/src/libutil/signature/signer.hh b/src/libutil/signature/signer.hh
new file mode 100644
index 000000000..e50170fe2
--- /dev/null
+++ b/src/libutil/signature/signer.hh
@@ -0,0 +1,61 @@
+#pragma once
+
+#include "types.hh"
+#include "signature/local-keys.hh"
+
+#include <map>
+#include <optional>
+
+namespace nix {
+
+/**
+ * An abstract signer
+ *
+ * Derive from this class to implement a custom signature scheme.
+ *
+ * It is only necessary to implement signature of bytes and provide a
+ * public key.
+ */
+struct Signer
+{
+    virtual ~Signer() = default;
+
+    /**
+     * Sign the given data, creating a (detached) signature.
+     *
+     * @param data data to be signed.
+     *
+     * @return the [detached
+     * signature](https://en.wikipedia.org/wiki/Detached_signature),
+     * i.e. just the signature itself without a copy of the signed data.
+     */
+    virtual std::string signDetached(std::string_view data) const = 0;
+
+    /**
+     * View the public key associated with this `Signer`.
+     */
+    virtual const PublicKey & getPublicKey() = 0;
+};
+
+using Signers = std::map<std::string, Signer*>;
+
+/**
+ * Local signer
+ *
+ * The private key is held in this machine's RAM
+ */
+struct LocalSigner : Signer
+{
+    LocalSigner(SecretKey && privateKey);
+
+    std::string signDetached(std::string_view s) const override;
+
+    const PublicKey & getPublicKey() override;
+
+private:
+
+    SecretKey privateKey;
+    PublicKey publicKey;
+};
+
+}
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 5bb3f374b..7b4b1d031 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -7,6 +7,7 @@
 #include <grp.h>
 #include <regex>
 
+#include <sodium.h>
 
 namespace nix {
 
@@ -28,6 +29,9 @@ void initLibUtil() {
     }
     // This is not actually the main point of this check, but let's make sure anyway:
     assert(caught);
+
+    if (sodium_init() == -1)
+        throw Error("could not initialise libsodium");
 }
 
 //////////////////////////////////////////////////////////////////////
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index a57a407e6..dfef44869 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -112,7 +112,7 @@ struct CmdSign : StorePathsCommand
 
     std::string description() override
     {
-        return "sign store paths";
+        return "sign store paths with a local key";
     }
 
     void run(ref<Store> store, StorePaths && storePaths) override
@@ -121,6 +121,7 @@ struct CmdSign : StorePathsCommand
             throw UsageError("you must specify a secret key file using '-k'");
 
         SecretKey secretKey(readFile(secretKeyFile));
+        LocalSigner signer(std::move(secretKey));
 
         size_t added{0};
 
@@ -129,7 +130,7 @@ struct CmdSign : StorePathsCommand
 
             auto info2(*info);
             info2.sigs.clear();
-            info2.sign(*store, secretKey);
+            info2.sign(*store, signer);
             assert(!info2.sigs.empty());
 
             if (!info->sigs.count(*info2.sigs.begin())) {
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index f0234f7be..2a0cbd19f 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -5,6 +5,7 @@
 #include "thread-pool.hh"
 #include "references.hh"
 #include "signals.hh"
+#include "keys.hh"
 
 #include <atomic>
 

From 37ea1612c78b88884f7baecbb1bf81e65e571592 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 3 Jan 2024 19:38:22 -0500
Subject: [PATCH 307/421] flake: Go back to regular `nixos-23.05-small`
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Finally get off the ad-hoc staging commit!

Flake lock file updates:

• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/36c4ac09e9bebcec1fa7b7539cddb0c9e837409c' (2023-11-30)
  → 'github:NixOS/nixpkgs/2c9c58e98243930f8cb70387934daa4bc8b00373' (2023-12-31)
---
 flake.lock              |  8 ++++----
 flake.nix               | 12 +-----------
 tests/nixos/default.nix |  1 -
 3 files changed, 5 insertions(+), 16 deletions(-)

diff --git a/flake.lock b/flake.lock
index db1a72c14..ae98d789a 100644
--- a/flake.lock
+++ b/flake.lock
@@ -34,16 +34,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1701355166,
-        "narHash": "sha256-4V7XMI0Gd+y0zsi++cEHd99u3GNL0xSTGRmiWKzGnUQ=",
+        "lastModified": 1704018918,
+        "narHash": "sha256-erjg/HrpC9liEfm7oLqb8GXCqsxaFwIIPqCsknW5aFY=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "36c4ac09e9bebcec1fa7b7539cddb0c9e837409c",
+        "rev": "2c9c58e98243930f8cb70387934daa4bc8b00373",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "staging-23.05",
+        "ref": "nixos-23.05-small",
         "repo": "nixpkgs",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index 9217de9af..e6a88af9f 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,17 +1,7 @@
 {
   description = "The purely functional package manager";
 
-  # TODO Go back to nixos-23.05-small once
-  # https://github.com/NixOS/nixpkgs/pull/271202 is merged.
-  #
-  # Also, do not grab arbitrary further staging commits. This PR was
-  # carefully made to be based on release-23.05 and just contain
-  # rebuild-causing changes to packages that Nix actually uses.
-  #
-  # Once this is updated to something containing
-  # https://github.com/NixOS/nixpkgs/pull/271423, don't forget
-  # to remove the `nix.checkAllErrors = false;` line in the tests.
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/staging-23.05";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
   inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; };
diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix
index 2645cac8e..4459aa664 100644
--- a/tests/nixos/default.nix
+++ b/tests/nixos/default.nix
@@ -10,7 +10,6 @@ let
     hostPkgs = nixpkgsFor.${system}.native;
     defaults = {
       nixpkgs.pkgs = nixpkgsFor.${system}.native;
-      nix.checkAllErrors = false;
     };
     _module.args.nixpkgs = nixpkgs;
   };

From d8a2b06e2068b5209264dfc6d74d5cadf88b8684 Mon Sep 17 00:00:00 2001
From: Rebecca Turner <rbt@sent.as>
Date: Thu, 4 Jan 2024 11:31:09 -0800
Subject: [PATCH 308/421] Remove `clang11Stdenv`

Clang 11 doesn't have support for three-way-comparisons (<=>, "spaceship
operator", "consistent comparisons") and is older than `clangStdenv`.

`clangStdenv` is currently 12 on FreeBSD and Android and 16 on other
platforms:

https://github.com/NixOS/nixpkgs/blob/32e718f00c26c811be0062dd0777066f02406940/pkgs/top-level/all-packages.nix#L16629-L16644

Let's start by removing Clang 11 from our distribution. Next we can
consider upgrading to Clang 17, which fully supports the spaceship
operator:

https://releases.llvm.org/17.0.1/tools/clang/docs/ReleaseNotes.html#what-s-new-in-clang-release
---
 doc/manual/src/contributing/hacking.md | 4 ++--
 flake.nix                              | 1 -
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index dce0422dc..9a03ac9b6 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -31,7 +31,7 @@ This shell also adds `./outputs/bin/nix` to your `$PATH` so you can run `nix` im
 To get a shell with one of the other [supported compilation environments](#compilation-environments):
 
 ```console
-$ nix develop .#native-clang11StdenvPackages
+$ nix develop .#native-clangStdenvPackages
 ```
 
 > **Note**
@@ -96,7 +96,7 @@ $ nix-shell
 To get a shell with one of the other [supported compilation environments](#compilation-environments):
 
 ```console
-$ nix-shell --attr devShells.x86_64-linux.native-clang11StdenvPackages
+$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages
 ```
 
 > **Note**
diff --git a/flake.nix b/flake.nix
index e6a88af9f..32354a88f 100644
--- a/flake.nix
+++ b/flake.nix
@@ -52,7 +52,6 @@
 
       stdenvs = [
         "ccacheStdenv"
-        "clang11Stdenv"
         "clangStdenv"
         "gccStdenv"
         "libcxxStdenv"

From 388c79d546db0a2e636aa56e4d4b9a5dfde50db5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 5 Jan 2024 15:15:25 +0100
Subject: [PATCH 309/421] Don't pull in libboost_regex

We're not using and we don't want to pull in libicu (37 MiB).
---
 package.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.nix b/package.nix
index 56276ecc4..dfebdb0e4 100644
--- a/package.nix
+++ b/package.nix
@@ -248,7 +248,7 @@ in {
       # Copy libboost_context so we don't get all of Boost in our closure.
       # https://github.com/NixOS/nixpkgs/issues/45462
       mkdir -p $out/lib
-      cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib
+      cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
       rm -f $out/lib/*.a
     '' + lib.optionalString stdenv.hostPlatform.isLinux ''
       chmod u+w $out/lib/*.so.*

From a4d33e816ef6c5baaed4eb65e826cd5aa75c0343 Mon Sep 17 00:00:00 2001
From: wiki-me <68199012+wiki-me@users.noreply.github.com>
Date: Sat, 6 Jan 2024 20:01:10 +0200
Subject: [PATCH 310/421] Improve documentation around upgrading nix (#9679)

* Improve documentation around upgrading nix, add replacing nix channel with new one

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 doc/manual/src/installation/upgrading.md | 35 ++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/doc/manual/src/installation/upgrading.md b/doc/manual/src/installation/upgrading.md
index 6d09f54d8..d1b64b80b 100644
--- a/doc/manual/src/installation/upgrading.md
+++ b/doc/manual/src/installation/upgrading.md
@@ -1,5 +1,40 @@
 # Upgrading Nix
 
+> **Note**
+>
+> These upgrade instructions apply for regular Linux distributions where Nix was installed following the [installation instructions in this manual](./index.md).
+
+First, find the name of the current [channel](@docroot@/command-ref/nix-channel.md) through which Nix is distributed:
+
+```console
+$ nix-channel --list
+```
+
+By default this should return an entry for Nixpkgs:
+
+```console
+nixpkgs https://nixos.org/channels/nixpkgs-23.05
+```
+
+Check which Nix version will be installed:
+
+```console
+$ nix-shell -p nix -I nixpkgs=channel:nixpkgs-23.11 --run "nix --version"
+nix (Nix) 2.18.1
+```
+
+> **Warning**
+>
+> Writing to the [local store](@docroot@/store/types/local-store.md) with a newer version of Nix, for example by building derivations with `nix-build` or `nix-store --realise`, may change the database schema!
+> Reverting to an older version of Nix may therefore require purging the store database before it can be used.
+
+Update the channel entry:
+
+```console
+$ nix-channel --remove nixpkgs
+$ nix-channel --add https://nixos.org/channels/nixpkgs-23.11 nixpkgs
+```
+
 Multi-user Nix users on macOS can upgrade Nix by running: `sudo -i sh -c
 'nix-channel --update &&
 nix-env --install --attr nixpkgs.nix &&

From 8e865f3aba526394ca333efe7258bd8db0050fbb Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Sat, 6 Jan 2024 22:45:25 +0100
Subject: [PATCH 311/421] deduplicate installation instructions (#9507)

* deduplicate installation instructions

- reorder sections to present pinned installation more prominently
- remove outdated notes on the macOS installer rework
- update instructions to handle the installer tarball

Co-authored-by: Travis A. Everett <travis.a.everett@gmail.com>
---
 .../src/installation/installing-binary.md     | 162 +++++++++---------
 doc/manual/src/quick-start.md                 |   1 -
 2 files changed, 77 insertions(+), 86 deletions(-)

diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md
index ffabb250a..0dc989159 100644
--- a/doc/manual/src/installation/installing-binary.md
+++ b/doc/manual/src/installation/installing-binary.md
@@ -1,26 +1,60 @@
 # Installing a Binary Distribution
 
-The easiest way to install Nix is to run the following command:
+To install the latest version Nix, run the following command:
 
 ```console
 $ curl -L https://nixos.org/nix/install | sh
 ```
 
-This will run the installer interactively (causing it to explain what
-it is doing more explicitly), and perform the default "type" of install
-for your platform:
-- single-user on Linux
-- multi-user on macOS
+This performs the default type of installation for your platform:
 
-  > **Notes on read-only filesystem root in macOS 10.15 Catalina +**
-  >
-  > - It took some time to support this cleanly. You may see posts,
-  >   examples, and tutorials using obsolete workarounds.
-  > - Supporting it cleanly made macOS installs too complex to qualify
-  >   as single-user, so this type is no longer supported on macOS.
+- [Multi-user](#multi-user-installation):
+  - Linux with systemd and without SELinux
+  - macOS
+- [Single-user](#single-user-installation):
+  - Linux without systemd
+  - Linux with SELinux
 
-We recommend the multi-user install if it supports your platform and
-you can authenticate with `sudo`.
+We recommend the multi-user installation if it supports your platform and you can authenticate with `sudo`.
+
+The installer can configured with various command line arguments and environment variables.
+To show available command line flags:
+
+```console
+$ curl -L https://nixos.org/nix/install | sh -s -- --help
+```
+
+To check what it does and how it can be customised further, [download and edit the second-stage installation script](#installing-from-a-binary-tarball).
+
+# Installing a pinned Nix version from a URL
+
+Version-specific installation URLs for all Nix versions since 1.11.16 can be found at [releases.nixos.org](https://releases.nixos.org/?prefix=nix/).
+The directory for each version contains the corresponding SHA-256 hash.
+
+All installation scripts are invoked the same way:
+
+```console
+$ export VERSION=2.19.2 
+$ curl -L https://releases.nixos.org/nix/nix-$VERSION/install | sh
+```
+
+# Multi User Installation
+
+The multi-user Nix installation creates system users and a system service for the Nix daemon.
+
+Supported systems:
+
+- Linux running systemd, with SELinux disabled
+- macOS
+
+To explicitly instruct the installer to perform a multi-user installation on your system:
+
+```console
+$ curl -L https://nixos.org/nix/install | sh -s -- --daemon
+```
+
+You can run this under your usual user account or `root`.
+The script will invoke `sudo` as needed.
 
 # Single User Installation
 
@@ -30,60 +64,48 @@ To explicitly select a single-user installation on your system:
 $ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon
 ```
 
-This will perform a single-user installation of Nix, meaning that `/nix`
-is owned by the invoking user. You can run this under your usual user
-account or root. The script will invoke `sudo` to create `/nix`
-if it doesn’t already exist. If you don’t have `sudo`, you should
-manually create `/nix` first as root, e.g.:
+In a single-user installation, `/nix` is owned by the invoking user.
+The script will invoke `sudo` to create `/nix` if it doesn’t already exist.
+If you don’t have `sudo`, manually create `/nix` as `root`:
 
 ```console
-$ mkdir /nix
-$ chown alice /nix
+$ su root
+# mkdir /nix
+# chown alice /nix
 ```
 
-The install script will modify the first writable file from amongst
-`.bash_profile`, `.bash_login` and `.profile` to source
-`~/.nix-profile/etc/profile.d/nix.sh`. You can set the
-`NIX_INSTALLER_NO_MODIFY_PROFILE` environment variable before executing
-the install script to disable this behaviour.
+# Installing from a binary tarball
 
-# Multi User Installation
+You can also download a binary tarball that contains Nix and all its dependencies:
+- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../contributing/hacking.md#platforms)
+- Download and unpack the tarball
+- Run the installer
 
-The multi-user Nix installation creates system users, and a system
-service for the Nix daemon.
-
-**Supported Systems**
-- Linux running systemd, with SELinux disabled
-- macOS
-
-You can instruct the installer to perform a multi-user installation on
-your system:
-
-```console
-$ curl -L https://nixos.org/nix/install | sh -s -- --daemon
-```
-
-The multi-user installation of Nix will create build users between the
-user IDs 30001 and 30032, and a group with the group ID 30000. You
-can run this under your usual user account or root. The script
-will invoke `sudo` as needed.
-
-> **Note**
+> **Example**
 >
-> If you need Nix to use a different group ID or user ID set, you will
-> have to download the tarball manually and [edit the install
-> script](#installing-from-a-binary-tarball).
+> ```console
+> $ pushd $(mktemp -d)
+> $ export VERSION=2.19.2
+> $ export SYSTEM=x86_64-linux
+> $ curl -LO https://releases.nixos.org/nix/nix-$VERSION/nix-$VERSION-$SYSTEM.tar.xz
+> $ tar xfj nix-$VERSION-$SYSTEM.tar.xz
+> $ cd nix-$VERSION-$SYSTEM
+> $ ./install
+> $ popd
+> ```
 
-The installer will modify `/etc/bashrc`, and `/etc/zshrc` if they exist.
-The installer will first back up these files with a `.backup-before-nix`
-extension. The installer will also create `/etc/profile.d/nix.sh`.
+The installer can be customised with the environment variables declared in the file named `install-multi-user`.
+
+## Native packages for Linux distributions
+
+The Nix community maintains installers for some Linux distributions in their native packaging format(https://nix-community.github.io/nix-installers/).
 
 # macOS Installation
 
+<!-- anchors to catch existing links -->
 []{#sect-macos-installation-change-store-prefix}[]{#sect-macos-installation-encrypted-volume}[]{#sect-macos-installation-symlink}[]{#sect-macos-installation-recommended-notes}
-<!-- Note: anchors above to catch permalinks to old explanations -->
 
-We believe we have ironed out how to cleanly support the read-only root
+We believe we have ironed out how to cleanly support the read-only root file system
 on modern macOS. New installs will do this automatically.
 
 This section previously detailed the situation, options, and trade-offs,
@@ -126,33 +148,3 @@ this to run the installer, but it may help if you run into trouble:
   boot process to avoid problems loading or restoring any programs that
   need access to your Nix store
 
-# Installing a pinned Nix version from a URL
-
-Version-specific installation URLs for all Nix versions
-since 1.11.16 can be found at [releases.nixos.org](https://releases.nixos.org/?prefix=nix/).
-The corresponding SHA-256 hash can be found in the directory for the given version.
-
-These install scripts can be used the same as usual:
-
-```console
-$ curl -L https://releases.nixos.org/nix/nix-<version>/install | sh
-```
-
-# Installing from a binary tarball
-
-You can also download a binary tarball that contains Nix and all its
-dependencies. (This is what the install script at
-<https://nixos.org/nix/install> does automatically.) You should unpack
-it somewhere (e.g. in `/tmp`), and then run the script named `install`
-inside the binary tarball:
-
-```console
-$ cd /tmp
-$ tar xfj nix-1.8-x86_64-darwin.tar.bz2
-$ cd nix-1.8-x86_64-darwin
-$ ./install
-```
-
-If you need to edit the multi-user installation script to use different
-group ID or a different user ID range, modify the variables set in the
-file named `install-multi-user`.
diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md
index 04a0b7c96..75853ced7 100644
--- a/doc/manual/src/quick-start.md
+++ b/doc/manual/src/quick-start.md
@@ -10,7 +10,6 @@ For more in-depth information you are kindly referred to subsequent chapters.
    ```
 
    The install script will use `sudo`, so make sure you have sufficient rights.
-   On Linux, `--daemon` can be omitted for a single-user install.
 
    For other installation methods, see the detailed [installation instructions](installation/index.md).
 

From fe751fbde22aea0362993ab7212f96630443c307 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Sat, 6 Jan 2024 23:44:15 +0100
Subject: [PATCH 312/421] don't show channels in upgrade instructions

channels make everything more stateful, and therefore more complicated
and potentially confusing, but aren't needed for this task, so don't encourage their use.
---
 doc/manual/src/installation/upgrading.md | 49 ++++++++++--------------
 1 file changed, 20 insertions(+), 29 deletions(-)

diff --git a/doc/manual/src/installation/upgrading.md b/doc/manual/src/installation/upgrading.md
index d1b64b80b..47618e2f5 100644
--- a/doc/manual/src/installation/upgrading.md
+++ b/doc/manual/src/installation/upgrading.md
@@ -2,48 +2,39 @@
 
 > **Note**
 >
-> These upgrade instructions apply for regular Linux distributions where Nix was installed following the [installation instructions in this manual](./index.md).
+> These upgrade instructions apply where Nix was installed following the [installation instructions in this manual](./index.md).
 
-First, find the name of the current [channel](@docroot@/command-ref/nix-channel.md) through which Nix is distributed:
+Check which Nix version will be installed, for example from one of the [release channels](http://channels.nixos.org/) such as `nixpkgs-unstable`:
 
 ```console
-$ nix-channel --list
-```
-
-By default this should return an entry for Nixpkgs:
-
-```console
-nixpkgs https://nixos.org/channels/nixpkgs-23.05
-```
-
-Check which Nix version will be installed:
-
-```console
-$ nix-shell -p nix -I nixpkgs=channel:nixpkgs-23.11 --run "nix --version"
+$ nix-shell -p nix -I nixpkgs=channel:nixpkgs-unstable --run "nix --version"
 nix (Nix) 2.18.1
 ```
 
 > **Warning**
 >
-> Writing to the [local store](@docroot@/store/types/local-store.md) with a newer version of Nix, for example by building derivations with `nix-build` or `nix-store --realise`, may change the database schema!
+> Writing to the [local store](@docroot@/store/types/local-store.md) with a newer version of Nix, for example by building derivations with [`nix-build`](@docroot@/command-ref/nix-build.md) or [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md), may change the database schema!
 > Reverting to an older version of Nix may therefore require purging the store database before it can be used.
 
-Update the channel entry:
+### Linux multi-user
 
 ```console
-$ nix-channel --remove nixpkgs
-$ nix-channel --add https://nixos.org/channels/nixpkgs-23.11 nixpkgs
+$ sudo su
+# nix-env --install --file '<nixpkgs>' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable
+# systemctl daemon-reload
+# systemctl restart nix-daemon
 ```
 
-Multi-user Nix users on macOS can upgrade Nix by running: `sudo -i sh -c
-'nix-channel --update &&
-nix-env --install --attr nixpkgs.nix &&
-launchctl remove org.nixos.nix-daemon &&
-launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist'`
+## macOS multi-user
 
-Single-user installations of Nix should run this: `nix-channel --update;
-nix-env --install --attr nixpkgs.nix nixpkgs.cacert`
+```console
+$ sudo nix-env --install --file '<nixpkgs>' --attr nix -I nixpkgs=channel:nixpkgs-unstable
+$ sudo launchctl remove org.nixos.nix-daemon
+$ sudo launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist
+```
 
-Multi-user Nix users on Linux should run this with sudo: `nix-channel
---update; nix-env --install --attr nixpkgs.nix nixpkgs.cacert; systemctl
-daemon-reload; systemctl restart nix-daemon`
+## Single-user all platforms
+
+```console
+$ nix-env --install --file '<nixpkgs>' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable
+```

From eeb2f083c5646bd3a66344cff69be586fd89a450 Mon Sep 17 00:00:00 2001
From: Shea Levy <shea@shealevy.com>
Date: Sun, 24 Dec 2023 06:44:56 -0500
Subject: [PATCH 313/421] Improve error message for fixed-outputs with
 references.

This codepath is possible, e.g. with a dockerTools.pullImage of an image with a Nix store.
---
 src/libstore/store-api.cc  | 5 ++++-
 tests/functional/fixed.nix | 9 +++++++++
 tests/functional/fixed.sh  | 3 +++
 3 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index c2516afb5..ad6e1cc0f 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -194,7 +194,10 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed
     if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::Recursive) {
         return makeStorePath(makeType(*this, "source", info.references), info.hash, name);
     } else {
-        assert(info.references.size() == 0);
+        if (!info.references.empty()) {
+            throw Error("fixed output derivation '%s' is not allowed to refer to other store paths.\nYou may need to use the 'unsafeDiscardReferences' derivation attribute, see the manual for more details.",
+                name);
+        }
         return makeStorePath("output:out",
             hashString(HashAlgorithm::SHA256,
                 "fixed:out:"
diff --git a/tests/functional/fixed.nix b/tests/functional/fixed.nix
index babe71504..5bdf79333 100644
--- a/tests/functional/fixed.nix
+++ b/tests/functional/fixed.nix
@@ -48,6 +48,15 @@ rec {
     (f ./fixed.builder1.sh "flat" "md5" "ddd8be4b179a529afa5f2ffae4b9858")
   ];
 
+  badReferences = mkDerivation rec {
+    name = "bad-hash";
+    builder = script;
+    script = builtins.toFile "installer.sh" "echo $script >$out";
+    outputHash = "1ixr6yd3297ciyp9im522dfxpqbkhcw0pylkb2aab915278fqaik";
+    outputHashAlgo = "sha256";
+    outputHashMode = "flat";
+  };
+
   # Test for building two derivations in parallel that produce the
   # same output path because they're fixed-output derivations.
   parallelSame = [
diff --git a/tests/functional/fixed.sh b/tests/functional/fixed.sh
index f1e1ce420..2405d059c 100644
--- a/tests/functional/fixed.sh
+++ b/tests/functional/fixed.sh
@@ -26,6 +26,9 @@ nix-build fixed.nix -A good2 --no-out-link
 echo 'testing reallyBad...'
 nix-instantiate fixed.nix -A reallyBad && fail "should fail"
 
+echo 'testing fixed with references...'
+expectStderr 1 nix-build fixed.nix -A badReferences | grepQuiet "not allowed to refer to other store paths"
+
 # While we're at it, check attribute selection a bit more.
 echo 'testing attribute selection...'
 test $(nix-instantiate fixed.nix -A good.1 | wc -l) = 1

From faf87b51f76ba9794e65e1d17dc3debf759052cd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 8 Jan 2024 14:14:36 +0100
Subject: [PATCH 314/421] Show why GC socket connection was refused

Co-authored-by: John Ericson <git@JohnEricson.me>
---
 src/libstore/gc.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index b5b9e2049..38a9c708b 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -141,7 +141,7 @@ void LocalStore::addTempRoot(const StorePath & path)
                 /* The garbage collector may have exited or not
                    created the socket yet, so we need to restart. */
                 if (e.errNo == ECONNREFUSED || e.errNo == ENOENT) {
-                    debug("GC socket connection refused");
+                    debug("GC socket connection refused: %s", e.msg())
                     fdRootsSocket->close();
                     goto restart;
                 }

From c4c636284e4b7b057788383068967910c5a31856 Mon Sep 17 00:00:00 2001
From: John Ericson <git@JohnEricson.me>
Date: Mon, 8 Jan 2024 10:17:28 -0500
Subject: [PATCH 315/421] Only test bug fix with new enough deamon

---
 tests/functional/fixed.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/functional/fixed.sh b/tests/functional/fixed.sh
index 2405d059c..d98d4cd15 100644
--- a/tests/functional/fixed.sh
+++ b/tests/functional/fixed.sh
@@ -26,8 +26,10 @@ nix-build fixed.nix -A good2 --no-out-link
 echo 'testing reallyBad...'
 nix-instantiate fixed.nix -A reallyBad && fail "should fail"
 
-echo 'testing fixed with references...'
-expectStderr 1 nix-build fixed.nix -A badReferences | grepQuiet "not allowed to refer to other store paths"
+if isDaemonNewer "2.20pre20240108"; then
+    echo 'testing fixed with references...'
+    expectStderr 1 nix-build fixed.nix -A badReferences | grepQuiet "not allowed to refer to other store paths"
+fi
 
 # While we're at it, check attribute selection a bit more.
 echo 'testing attribute selection...'

From 605eba3829946eb04f1aaf1160cf11a55183c677 Mon Sep 17 00:00:00 2001
From: Weijia Wang <9713184+wegank@users.noreply.github.com>
Date: Mon, 8 Jan 2024 17:31:27 +0100
Subject: [PATCH 316/421] Fix typo in configure.ac

---
 configure.ac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index b97e25bbd..369d62552 100644
--- a/configure.ac
+++ b/configure.ac
@@ -160,7 +160,7 @@ AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation
 AC_SUBST(ENABLE_DOC_GEN)
 
 AS_IF(
-  [test "$ENABLE_BUILD" == "no" && test "$ENABLE_GENERATED_DOCS" == "yes"],
+  [test "$ENABLE_BUILD" == "no" && test "$ENABLE_DOC_GEN" == "yes"],
   [AC_MSG_ERROR([Cannot enable generated docs when building overall is disabled. Please do not pass '--enable-doc-gen' or do not pass '--disable-build'.])])
 
 # Building without API docs is the default as Nix' C++ interfaces are internal and unstable.

From 6a243e5ed281344135285d9093ef36969a867d73 Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Mon, 8 Jan 2024 19:38:36 +0100
Subject: [PATCH 317/421] fix an old lost direct (#9458)

this part must have been moved quite a while ago, but apparently so far
no one noticed
---
 doc/manual/redirects.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js
index 3b507adf3..d04f32b49 100644
--- a/doc/manual/redirects.js
+++ b/doc/manual/redirects.js
@@ -21,6 +21,7 @@ const redirects = {
     "chap-distributed-builds": "advanced-topics/distributed-builds.html",
     "chap-post-build-hook": "advanced-topics/post-build-hook.html",
     "chap-post-build-hook-caveats": "advanced-topics/post-build-hook.html#implementation-caveats",
+    "chap-writing-nix-expressions": "language/index.html",
     "part-command-ref": "command-ref/command-ref.html",
     "conf-allow-import-from-derivation": "command-ref/conf-file.html#conf-allow-import-from-derivation",
     "conf-allow-new-privileges": "command-ref/conf-file.html#conf-allow-new-privileges",

From 53fdcbca509b6c5dacaea3d3c465d86e49b0dd74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20Luis=20Lafuente?= <jl@lafuente.dev>
Date: Mon, 8 Jan 2024 19:46:38 +0100
Subject: [PATCH 318/421] Add clang format configuration

---
 .clang-format | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 .clang-format

diff --git a/.clang-format b/.clang-format
new file mode 100644
index 000000000..9c0c0946a
--- /dev/null
+++ b/.clang-format
@@ -0,0 +1,30 @@
+BasedOnStyle: LLVM
+IndentWidth: 4
+BreakBeforeBraces: Custom
+BraceWrapping:
+  AfterStruct: true
+  AfterClass: true
+  AfterFunction: true
+  AfterUnion: true
+  SplitEmptyRecord: false
+PointerAlignment: Middle
+FixNamespaceComments: false
+SortIncludes: Never
+#IndentPPDirectives: BeforeHash
+SpaceAfterCStyleCast: true
+SpaceAfterTemplateKeyword: false
+AccessModifierOffset: -4
+AlignAfterOpenBracket: AlwaysBreak
+AlignEscapedNewlines: DontAlign
+ColumnLimit: 120
+BreakStringLiterals: false
+BitFieldColonSpacing: None
+AllowShortFunctionsOnASingleLine: Empty
+AlwaysBreakTemplateDeclarations: Yes
+BinPackParameters: false
+BreakConstructorInitializers: BeforeComma
+EmptyLineAfterAccessModifier: Leave # change to always/never later?
+EmptyLineBeforeAccessModifier: Leave
+#PackConstructorInitializers: BinPack
+BreakBeforeBinaryOperators: NonAssignment
+AlwaysBreakBeforeMultilineStrings: true

From 4feb7d9f715021784952bea57b37a8628c9b6860 Mon Sep 17 00:00:00 2001
From: Rebecca Turner <rbt@sent.as>
Date: Mon, 18 Dec 2023 13:14:42 -0800
Subject: [PATCH 319/421] Combine `AbstractPos`, `PosAdapter`, and `Pos`

Also move `SourcePath` into `libutil`.

These changes allow `error.hh` and `error.cc` to access source path and
position information, which we can use to produce better error messages
(for example, we could consider omitting filenames when two or more
consecutive stack frames originate from the same file).
---
 src/libcmd/editor-for.cc                    |   1 +
 src/libcmd/editor-for.hh                    |   2 +-
 src/libcmd/installable-value.cc             |   3 +-
 src/libcmd/repl.cc                          |   2 +-
 src/libexpr/eval.cc                         |   9 +-
 src/libexpr/eval.hh                         |   2 +-
 src/libexpr/nixexpr.cc                      |  63 -------
 src/libexpr/nixexpr.hh                      |  26 +--
 src/libexpr/primops.cc                      |   3 +-
 src/libexpr/value.hh                        |   1 +
 src/libfetchers/fetch-to-store.cc           |  68 ++++++++
 src/libfetchers/fetch-to-store.hh           |  22 +++
 src/libfetchers/fetchers.cc                 |   4 +-
 src/libfetchers/filtering-input-accessor.hh |   1 +
 src/libfetchers/fs-input-accessor.hh        |   1 +
 src/libfetchers/input-accessor.cc           | 129 ---------------
 src/libfetchers/input-accessor.hh           | 174 --------------------
 src/libfetchers/memory-input-accessor.cc    |   1 +
 src/libfetchers/memory-input-accessor.hh    |   1 +
 src/libstore/store-api.hh                   |   1 +
 src/libutil/error.cc                        |  55 +------
 src/libutil/error.hh                        |  42 +----
 src/libutil/input-accessor.hh               |  27 +++
 src/libutil/logging.cc                      |   6 +-
 src/libutil/position.cc                     | 112 +++++++++++++
 src/libutil/position.hh                     |  74 +++++++++
 src/libutil/ref.hh                          |   1 +
 src/{libstore => libutil}/repair-flag.hh    |   0
 src/libutil/source-path.cc                  | 105 ++++++++++++
 src/libutil/source-path.hh                  | 114 +++++++++++++
 30 files changed, 561 insertions(+), 489 deletions(-)
 create mode 100644 src/libfetchers/fetch-to-store.cc
 create mode 100644 src/libfetchers/fetch-to-store.hh
 delete mode 100644 src/libfetchers/input-accessor.cc
 delete mode 100644 src/libfetchers/input-accessor.hh
 create mode 100644 src/libutil/input-accessor.hh
 create mode 100644 src/libutil/position.cc
 create mode 100644 src/libutil/position.hh
 rename src/{libstore => libutil}/repair-flag.hh (100%)
 create mode 100644 src/libutil/source-path.cc
 create mode 100644 src/libutil/source-path.hh

diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc
index 619d3673f..67653d9c9 100644
--- a/src/libcmd/editor-for.cc
+++ b/src/libcmd/editor-for.cc
@@ -1,5 +1,6 @@
 #include "editor-for.hh"
 #include "environment-variables.hh"
+#include "source-path.hh"
 
 namespace nix {
 
diff --git a/src/libcmd/editor-for.hh b/src/libcmd/editor-for.hh
index fbf4307c9..8acd7011e 100644
--- a/src/libcmd/editor-for.hh
+++ b/src/libcmd/editor-for.hh
@@ -2,7 +2,7 @@
 ///@file
 
 #include "types.hh"
-#include "input-accessor.hh"
+#include "source-path.hh"
 
 namespace nix {
 
diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc
index bdc34bbe3..c8a3e1b21 100644
--- a/src/libcmd/installable-value.cc
+++ b/src/libcmd/installable-value.cc
@@ -1,5 +1,6 @@
 #include "installable-value.hh"
 #include "eval-cache.hh"
+#include "fetch-to-store.hh"
 
 namespace nix {
 
@@ -44,7 +45,7 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
 std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
 {
     if (v.type() == nPath) {
-        auto storePath = v.path().fetchToStore(*state->store);
+        auto storePath = fetchToStore(*state->store, v.path());
         return {{
             .path = DerivedPath::Opaque {
                 .path = std::move(storePath),
diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc
index dea91ba63..78c4538b2 100644
--- a/src/libcmd/repl.cc
+++ b/src/libcmd/repl.cc
@@ -221,7 +221,7 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi
     // prefer direct pos, but if noPos then try the expr.
     auto pos = dt.pos
         ? dt.pos
-        : static_cast<std::shared_ptr<AbstractPos>>(positions[dt.expr.getPos() ? dt.expr.getPos() : noPos]);
+        : positions[dt.expr.getPos() ? dt.expr.getPos() : noPos];
 
     if (pos) {
         out << pos;
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 31f2d4952..d408f1adc 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -19,6 +19,7 @@
 #include "signals.hh"
 #include "gc-small-vector.hh"
 #include "url.hh"
+#include "fetch-to-store.hh"
 
 #include <algorithm>
 #include <chrono>
@@ -870,7 +871,7 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr &
         ? std::make_unique<DebugTraceStacker>(
             *this,
             DebugTrace {
-                .pos = error->info().errPos ? error->info().errPos : static_cast<std::shared_ptr<AbstractPos>>(positions[expr.getPos()]),
+                .pos = error->info().errPos ? error->info().errPos : positions[expr.getPos()],
                 .expr = expr,
                 .env = env,
                 .hint = error->info().msg,
@@ -909,7 +910,7 @@ static std::unique_ptr<DebugTraceStacker> makeDebugTraceStacker(
     EvalState & state,
     Expr & expr,
     Env & env,
-    std::shared_ptr<AbstractPos> && pos,
+    std::shared_ptr<Pos> && pos,
     const char * s,
     const std::string & s2)
 {
@@ -1187,7 +1188,7 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial)
                 *this,
                 *e,
                 this->baseEnv,
-                e->getPos() ? static_cast<std::shared_ptr<AbstractPos>>(positions[e->getPos()]) : nullptr,
+                e->getPos() ? std::make_shared<Pos>(positions[e->getPos()]) : nullptr,
                 "while evaluating the file '%1%':", resolvedPath.to_string())
             : nullptr;
 
@@ -2368,7 +2369,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
     auto dstPath = i != srcToStore.end()
         ? i->second
         : [&]() {
-            auto dstPath = path.fetchToStore(*store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair);
+            auto dstPath = fetchToStore(*store, path, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair);
             allowPath(dstPath);
             srcToStore.insert_or_assign(path, dstPath);
             printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 6e3f08d55..5e0f1886d 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -142,7 +142,7 @@ struct RegexCache;
 std::shared_ptr<RegexCache> makeRegexCache();
 
 struct DebugTrace {
-    std::shared_ptr<AbstractPos> pos;
+    std::shared_ptr<Pos> pos;
     const Expr & expr;
     const Env & env;
     hintformat hint;
diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc
index ede070cff..964de6351 100644
--- a/src/libexpr/nixexpr.cc
+++ b/src/libexpr/nixexpr.cc
@@ -11,58 +11,6 @@ namespace nix {
 
 ExprBlackHole eBlackHole;
 
-struct PosAdapter : AbstractPos
-{
-    Pos::Origin origin;
-
-    PosAdapter(Pos::Origin origin)
-        : origin(std::move(origin))
-    {
-    }
-
-    std::optional<std::string> getSource() const override
-    {
-        return std::visit(overloaded {
-            [](const Pos::none_tag &) -> std::optional<std::string> {
-                return std::nullopt;
-            },
-            [](const Pos::Stdin & s) -> std::optional<std::string> {
-                // Get rid of the null terminators added by the parser.
-                return std::string(s.source->c_str());
-            },
-            [](const Pos::String & s) -> std::optional<std::string> {
-                // Get rid of the null terminators added by the parser.
-                return std::string(s.source->c_str());
-            },
-            [](const SourcePath & path) -> std::optional<std::string> {
-                try {
-                    return path.readFile();
-                } catch (Error &) {
-                    return std::nullopt;
-                }
-            }
-        }, origin);
-    }
-
-    void print(std::ostream & out) const override
-    {
-        std::visit(overloaded {
-            [&](const Pos::none_tag &) { out << "«none»"; },
-            [&](const Pos::Stdin &) { out << "«stdin»"; },
-            [&](const Pos::String & s) { out << "«string»"; },
-            [&](const SourcePath & path) { out << path; }
-        }, origin);
-    }
-};
-
-Pos::operator std::shared_ptr<AbstractPos>() const
-{
-    auto pos = std::make_shared<PosAdapter>(origin);
-    pos->line = line;
-    pos->column = column;
-    return pos;
-}
-
 // FIXME: remove, because *symbols* are abstract and do not have a single
 //        textual representation; see printIdentifier()
 std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol)
@@ -268,17 +216,6 @@ void ExprPos::show(const SymbolTable & symbols, std::ostream & str) const
 }
 
 
-std::ostream & operator << (std::ostream & str, const Pos & pos)
-{
-    if (auto pos2 = (std::shared_ptr<AbstractPos>) pos) {
-        str << *pos2;
-    } else
-        str << "undefined position";
-
-    return str;
-}
-
-
 std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath)
 {
     std::ostringstream out;
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index 71ed9ef30..3cd46ca27 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -8,6 +8,7 @@
 #include "symbol-table.hh"
 #include "error.hh"
 #include "chunked-vector.hh"
+#include "position.hh"
 
 namespace nix {
 
@@ -28,27 +29,6 @@ public:
     using EvalError::EvalError;
 };
 
-/**
- * Position objects.
- */
-struct Pos
-{
-    uint32_t line;
-    uint32_t column;
-
-    struct none_tag { };
-    struct Stdin { ref<std::string> source; };
-    struct String { ref<std::string> source; };
-
-    typedef std::variant<none_tag, Stdin, String, SourcePath> Origin;
-
-    Origin origin;
-
-    explicit operator bool() const { return line > 0; }
-
-    operator std::shared_ptr<AbstractPos>() const;
-};
-
 class PosIdx {
     friend class PosTable;
 
@@ -81,7 +61,7 @@ public:
         mutable uint32_t idx = std::numeric_limits<uint32_t>::max();
 
         // Used for searching in PosTable::[].
-        explicit Origin(uint32_t idx): idx(idx), origin{Pos::none_tag()} {}
+        explicit Origin(uint32_t idx): idx(idx), origin{std::monostate()} {}
 
     public:
         const Pos::Origin origin;
@@ -132,8 +112,6 @@ public:
 
 inline PosIdx noPos = {};
 
-std::ostream & operator << (std::ostream & str, const Pos & pos);
-
 
 struct Env;
 struct Value;
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index b2ffcc051..ee07e5568 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -16,6 +16,7 @@
 #include "value-to-xml.hh"
 #include "primops.hh"
 #include "fs-input-accessor.hh"
+#include "fetch-to-store.hh"
 
 #include <boost/container/small_vector.hpp>
 #include <nlohmann/json.hpp>
@@ -2240,7 +2241,7 @@ static void addPath(
             });
 
         if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
-            auto dstPath = path.fetchToStore(*state.store, name, method, filter.get(), state.repair);
+            auto dstPath = fetchToStore(*state.store, path, name, method, filter.get(), state.repair);
             if (expectedHash && expectedStorePath != dstPath)
                 state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path));
             state.allowAndSetStorePathString(dstPath, v);
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index d9860e921..c65b336b0 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -8,6 +8,7 @@
 #include "symbol-table.hh"
 #include "value/context.hh"
 #include "input-accessor.hh"
+#include "source-path.hh"
 
 #if HAVE_BOEHMGC
 #include <gc/gc_allocator.h>
diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc
new file mode 100644
index 000000000..196489e05
--- /dev/null
+++ b/src/libfetchers/fetch-to-store.cc
@@ -0,0 +1,68 @@
+#include "fetch-to-store.hh"
+#include "fetchers.hh"
+#include "cache.hh"
+
+namespace nix {
+
+StorePath fetchToStore(
+    Store & store,
+    const SourcePath & path,
+    std::string_view name,
+    ContentAddressMethod method,
+    PathFilter * filter,
+    RepairFlag repair)
+{
+    // FIXME: add an optimisation for the case where the accessor is
+    // an FSInputAccessor pointing to a store path.
+
+    std::optional<fetchers::Attrs> cacheKey;
+
+    if (!filter && path.accessor->fingerprint) {
+        cacheKey = fetchers::Attrs{
+            {"_what", "fetchToStore"},
+            {"store", store.storeDir},
+            {"name", std::string(name)},
+            {"fingerprint", *path.accessor->fingerprint},
+            {
+                "method",
+                std::visit(overloaded {
+                    [](const TextIngestionMethod &) {
+                        return "text";
+                    },
+                    [](const FileIngestionMethod & fim) {
+                        switch (fim) {
+                        case FileIngestionMethod::Flat: return "flat";
+                        case FileIngestionMethod::Recursive: return "nar";
+                        default: assert(false);
+                        }
+                    },
+                }, method.raw),
+            },
+            {"path", path.path.abs()}
+        };
+        if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) {
+            debug("store path cache hit for '%s'", path);
+            return res->second;
+        }
+    } else
+        debug("source path '%s' is uncacheable", path);
+
+    Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", path));
+
+    auto filter2 = filter ? *filter : defaultPathFilter;
+
+    auto storePath =
+        settings.readOnlyMode
+        ? store.computeStorePath(
+            name, *path.accessor, path.path, method, HashAlgorithm::SHA256, {}, filter2).first
+        : store.addToStore(
+            name, *path.accessor, path.path, method, HashAlgorithm::SHA256, {}, filter2, repair);
+
+    if (cacheKey)
+        fetchers::getCache()->add(store, *cacheKey, {}, storePath, true);
+
+    return storePath;
+}
+
+
+}
diff --git a/src/libfetchers/fetch-to-store.hh b/src/libfetchers/fetch-to-store.hh
new file mode 100644
index 000000000..e5e039340
--- /dev/null
+++ b/src/libfetchers/fetch-to-store.hh
@@ -0,0 +1,22 @@
+#pragma once
+
+#include "source-path.hh"
+#include "store-api.hh"
+#include "file-system.hh"
+#include "repair-flag.hh"
+#include "file-content-address.hh"
+
+namespace nix {
+
+/**
+ * Copy the `path` to the Nix store.
+ */
+StorePath fetchToStore(
+    Store & store,
+    const SourcePath & path,
+    std::string_view name = "source",
+    ContentAddressMethod method = FileIngestionMethod::Recursive,
+    PathFilter * filter = nullptr,
+    RepairFlag repair = NoRepair);
+
+}
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index f309e5993..7f282c972 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -1,6 +1,8 @@
 #include "fetchers.hh"
 #include "store-api.hh"
 #include "input-accessor.hh"
+#include "source-path.hh"
+#include "fetch-to-store.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -374,7 +376,7 @@ void InputScheme::clone(const Input & input, const Path & destDir) const
 std::pair<StorePath, Input> InputScheme::fetch(ref<Store> store, const Input & input)
 {
     auto [accessor, input2] = getAccessor(store, input);
-    auto storePath = SourcePath(accessor).fetchToStore(*store, input2.getName());
+    auto storePath = fetchToStore(*store, SourcePath(accessor), input2.getName());
     return {storePath, input2};
 }
 
diff --git a/src/libfetchers/filtering-input-accessor.hh b/src/libfetchers/filtering-input-accessor.hh
index e1b83c929..a352a33a6 100644
--- a/src/libfetchers/filtering-input-accessor.hh
+++ b/src/libfetchers/filtering-input-accessor.hh
@@ -1,6 +1,7 @@
 #pragma once
 
 #include "input-accessor.hh"
+#include "source-path.hh"
 
 namespace nix {
 
diff --git a/src/libfetchers/fs-input-accessor.hh b/src/libfetchers/fs-input-accessor.hh
index ba5af5887..a98e83511 100644
--- a/src/libfetchers/fs-input-accessor.hh
+++ b/src/libfetchers/fs-input-accessor.hh
@@ -1,6 +1,7 @@
 #pragma once
 
 #include "input-accessor.hh"
+#include "source-path.hh"
 
 namespace nix {
 
diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc
deleted file mode 100644
index a647f5915..000000000
--- a/src/libfetchers/input-accessor.cc
+++ /dev/null
@@ -1,129 +0,0 @@
-#include "input-accessor.hh"
-#include "store-api.hh"
-#include "cache.hh"
-
-namespace nix {
-
-StorePath InputAccessor::fetchToStore(
-    Store & store,
-    const CanonPath & path,
-    std::string_view name,
-    ContentAddressMethod method,
-    PathFilter * filter,
-    RepairFlag repair)
-{
-    // FIXME: add an optimisation for the case where the accessor is
-    // an FSInputAccessor pointing to a store path.
-
-    std::optional<fetchers::Attrs> cacheKey;
-
-    if (!filter && fingerprint) {
-        cacheKey = fetchers::Attrs{
-            {"_what", "fetchToStore"},
-            {"store", store.storeDir},
-            {"name", std::string(name)},
-            {"fingerprint", *fingerprint},
-            {
-                "method",
-                std::visit(overloaded {
-                    [](const TextIngestionMethod &) {
-                        return "text";
-                    },
-                    [](const FileIngestionMethod & fim) {
-                        switch (fim) {
-                        case FileIngestionMethod::Flat: return "flat";
-                        case FileIngestionMethod::Recursive: return "nar";
-                        default: assert(false);
-                        }
-                    },
-                }, method.raw),
-            },
-            {"path", path.abs()}
-        };
-        if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) {
-            debug("store path cache hit for '%s'", showPath(path));
-            return res->second;
-        }
-    } else
-        debug("source path '%s' is uncacheable", showPath(path));
-
-    Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path)));
-
-    auto filter2 = filter ? *filter : defaultPathFilter;
-
-    auto storePath =
-        settings.readOnlyMode
-        ? store.computeStorePath(
-            name, *this, path, method, HashAlgorithm::SHA256, {}, filter2).first
-        : store.addToStore(
-            name, *this, path, method, HashAlgorithm::SHA256, {}, filter2, repair);
-
-    if (cacheKey)
-        fetchers::getCache()->add(store, *cacheKey, {}, storePath, true);
-
-    return storePath;
-}
-
-std::ostream & operator << (std::ostream & str, const SourcePath & path)
-{
-    str << path.to_string();
-    return str;
-}
-
-StorePath SourcePath::fetchToStore(
-    Store & store,
-    std::string_view name,
-    ContentAddressMethod method,
-    PathFilter * filter,
-    RepairFlag repair) const
-{
-    return accessor->fetchToStore(store, path, name, method, filter, repair);
-}
-
-std::string_view SourcePath::baseName() const
-{
-    return path.baseName().value_or("source");
-}
-
-SourcePath SourcePath::parent() const
-{
-    auto p = path.parent();
-    assert(p);
-    return {accessor, std::move(*p)};
-}
-
-SourcePath SourcePath::resolveSymlinks() const
-{
-    auto res = SourcePath(accessor);
-
-    int linksAllowed = 1024;
-
-    std::list<std::string> todo;
-    for (auto & c : path)
-        todo.push_back(std::string(c));
-
-    while (!todo.empty()) {
-        auto c = *todo.begin();
-        todo.pop_front();
-        if (c == "" || c == ".")
-            ;
-        else if (c == "..")
-            res.path.pop();
-        else {
-            res.path.push(c);
-            if (auto st = res.maybeLstat(); st && st->type == InputAccessor::tSymlink) {
-                if (!linksAllowed--)
-                    throw Error("infinite symlink recursion in path '%s'", path);
-                auto target = res.readLink();
-                res.path.pop();
-                if (hasPrefix(target, "/"))
-                    res.path = CanonPath::root;
-                todo.splice(todo.begin(), tokenizeString<std::list<std::string>>(target, "/"));
-            }
-        }
-    }
-
-    return res;
-}
-
-}
diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh
deleted file mode 100644
index d2a21cb4b..000000000
--- a/src/libfetchers/input-accessor.hh
+++ /dev/null
@@ -1,174 +0,0 @@
-#pragma once
-///@file
-
-#include "source-accessor.hh"
-#include "ref.hh"
-#include "types.hh"
-#include "file-system.hh"
-#include "repair-flag.hh"
-#include "content-address.hh"
-
-namespace nix {
-
-MakeError(RestrictedPathError, Error);
-
-struct SourcePath;
-class StorePath;
-class Store;
-
-struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this<InputAccessor>
-{
-    std::optional<std::string> fingerprint;
-
-    /**
-     * Return the maximum last-modified time of the files in this
-     * tree, if available.
-     */
-    virtual std::optional<time_t> getLastModified()
-    {
-        return std::nullopt;
-    }
-
-    StorePath fetchToStore(
-        Store & store,
-        const CanonPath & path,
-        std::string_view name = "source",
-        ContentAddressMethod method = FileIngestionMethod::Recursive,
-        PathFilter * filter = nullptr,
-        RepairFlag repair = NoRepair);
-};
-
-/**
- * An abstraction for accessing source files during
- * evaluation. Currently, it's just a wrapper around `CanonPath` that
- * accesses files in the regular filesystem, but in the future it will
- * support fetching files in other ways.
- */
-struct SourcePath
-{
-    ref<InputAccessor> accessor;
-    CanonPath path;
-
-    SourcePath(ref<InputAccessor> accessor, CanonPath path = CanonPath::root)
-        : accessor(std::move(accessor))
-        , path(std::move(path))
-    { }
-
-    std::string_view baseName() const;
-
-    /**
-     * Construct the parent of this `SourcePath`. Aborts if `this`
-     * denotes the root.
-     */
-    SourcePath parent() const;
-
-    /**
-     * If this `SourcePath` denotes a regular file (not a symlink),
-     * return its contents; otherwise throw an error.
-     */
-    std::string readFile() const
-    { return accessor->readFile(path); }
-
-    /**
-     * Return whether this `SourcePath` denotes a file (of any type)
-     * that exists
-    */
-    bool pathExists() const
-    { return accessor->pathExists(path); }
-
-    /**
-     * Return stats about this `SourcePath`, or throw an exception if
-     * it doesn't exist.
-     */
-    InputAccessor::Stat lstat() const
-    { return accessor->lstat(path); }
-
-    /**
-     * Return stats about this `SourcePath`, or std::nullopt if it
-     * doesn't exist.
-     */
-    std::optional<InputAccessor::Stat> maybeLstat() const
-    { return accessor->maybeLstat(path); }
-
-    /**
-     * If this `SourcePath` denotes a directory (not a symlink),
-     * return its directory entries; otherwise throw an error.
-     */
-    InputAccessor::DirEntries readDirectory() const
-    { return accessor->readDirectory(path); }
-
-    /**
-     * If this `SourcePath` denotes a symlink, return its target;
-     * otherwise throw an error.
-     */
-    std::string readLink() const
-    { return accessor->readLink(path); }
-
-    /**
-     * Dump this `SourcePath` to `sink` as a NAR archive.
-     */
-    void dumpPath(
-        Sink & sink,
-        PathFilter & filter = defaultPathFilter) const
-    { return accessor->dumpPath(path, sink, filter); }
-
-    /**
-     * Copy this `SourcePath` to the Nix store.
-     */
-    StorePath fetchToStore(
-        Store & store,
-        std::string_view name = "source",
-        ContentAddressMethod method = FileIngestionMethod::Recursive,
-        PathFilter * filter = nullptr,
-        RepairFlag repair = NoRepair) const;
-
-    /**
-     * Return the location of this path in the "real" filesystem, if
-     * it has a physical location.
-     */
-    std::optional<CanonPath> getPhysicalPath() const
-    { return accessor->getPhysicalPath(path); }
-
-    std::string to_string() const
-    { return accessor->showPath(path); }
-
-    /**
-     * Append a `CanonPath` to this path.
-     */
-    SourcePath operator + (const CanonPath & x) const
-    { return {accessor, path + x}; }
-
-    /**
-     * Append a single component `c` to this path. `c` must not
-     * contain a slash. A slash is implicitly added between this path
-     * and `c`.
-     */
-    SourcePath operator + (std::string_view c) const
-    {  return {accessor, path + c}; }
-
-    bool operator == (const SourcePath & x) const
-    {
-        return std::tie(accessor, path) == std::tie(x.accessor, x.path);
-    }
-
-    bool operator != (const SourcePath & x) const
-    {
-        return std::tie(accessor, path) != std::tie(x.accessor, x.path);
-    }
-
-    bool operator < (const SourcePath & x) const
-    {
-        return std::tie(accessor, path) < std::tie(x.accessor, x.path);
-    }
-
-    /**
-     * Resolve any symlinks in this `SourcePath` (including its
-     * parents). The result is a `SourcePath` in which no element is a
-     * symlink.
-     */
-    SourcePath resolveSymlinks() const;
-};
-
-std::ostream & operator << (std::ostream & str, const SourcePath & path);
-
-}
diff --git a/src/libfetchers/memory-input-accessor.cc b/src/libfetchers/memory-input-accessor.cc
index 057f3e37f..88a2e34e8 100644
--- a/src/libfetchers/memory-input-accessor.cc
+++ b/src/libfetchers/memory-input-accessor.cc
@@ -1,5 +1,6 @@
 #include "memory-input-accessor.hh"
 #include "memory-source-accessor.hh"
+#include "source-path.hh"
 
 namespace nix {
 
diff --git a/src/libfetchers/memory-input-accessor.hh b/src/libfetchers/memory-input-accessor.hh
index b75b02bfd..508b07722 100644
--- a/src/libfetchers/memory-input-accessor.hh
+++ b/src/libfetchers/memory-input-accessor.hh
@@ -1,4 +1,5 @@
 #include "input-accessor.hh"
+#include "source-path.hh"
 
 namespace nix {
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 96a7ebd7b..9667b5e9e 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -13,6 +13,7 @@
 #include "path-info.hh"
 #include "repair-flag.hh"
 #include "store-dir-config.hh"
+#include "source-path.hh"
 
 #include <nlohmann/json_fwd.hpp>
 #include <atomic>
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index e42925c2b..bd2f6b840 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -2,6 +2,7 @@
 #include "environment-variables.hh"
 #include "signals.hh"
 #include "terminal.hh"
+#include "position.hh"
 
 #include <iostream>
 #include <optional>
@@ -10,7 +11,7 @@
 
 namespace nix {
 
-void BaseError::addTrace(std::shared_ptr<AbstractPos> && e, hintformat hint, bool frame)
+void BaseError::addTrace(std::shared_ptr<Pos> && e, hintformat hint, bool frame)
 {
     err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .frame = frame });
 }
@@ -41,15 +42,6 @@ std::ostream & operator <<(std::ostream & os, const hintformat & hf)
     return os << hf.str();
 }
 
-std::ostream & operator <<(std::ostream & str, const AbstractPos & pos)
-{
-    pos.print(str);
-    str << ":" << pos.line;
-    if (pos.column > 0)
-        str << ":" << pos.column;
-    return str;
-}
-
 /**
  * An arbitrarily defined value comparison for the purpose of using traces in the key of a sorted container.
  */
@@ -76,49 +68,10 @@ inline bool operator> (const Trace& lhs, const Trace& rhs) { return rhs < lhs; }
 inline bool operator<=(const Trace& lhs, const Trace& rhs) { return !(lhs > rhs); }
 inline bool operator>=(const Trace& lhs, const Trace& rhs) { return !(lhs < rhs); }
 
-std::optional<LinesOfCode> AbstractPos::getCodeLines() const
-{
-    if (line == 0)
-        return std::nullopt;
-
-    if (auto source = getSource()) {
-
-        std::istringstream iss(*source);
-        // count the newlines.
-        int count = 0;
-        std::string curLine;
-        int pl = line - 1;
-
-        LinesOfCode loc;
-
-        do {
-            std::getline(iss, curLine);
-            ++count;
-            if (count < pl)
-                ;
-            else if (count == pl) {
-                loc.prevLineOfCode = curLine;
-            } else if (count == pl + 1) {
-                loc.errLineOfCode = curLine;
-            } else if (count == pl + 2) {
-                loc.nextLineOfCode = curLine;
-                break;
-            }
-
-            if (!iss.good())
-                break;
-        } while (true);
-
-        return loc;
-    }
-
-    return std::nullopt;
-}
-
 // print lines of code to the ostream, indicating the error column.
 void printCodeLines(std::ostream & out,
     const std::string & prefix,
-    const AbstractPos & errPos,
+    const Pos & errPos,
     const LinesOfCode & loc)
 {
     // previous line of code.
@@ -196,7 +149,7 @@ static bool printUnknownLocations = getEnv("_NIX_EVAL_SHOW_UNKNOWN_LOCATIONS").h
  *
  * @return true if a position was printed.
  */
-static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std::shared_ptr<AbstractPos> & pos) {
+static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std::shared_ptr<Pos> & pos) {
     bool hasPos = pos && *pos;
     if (hasPos) {
         oss << indent << ANSI_BLUE << "at " ANSI_WARNING << *pos << ANSI_NORMAL << ":";
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index baffca128..234cbe1f6 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -63,45 +63,15 @@ struct LinesOfCode {
     std::optional<std::string> nextLineOfCode;
 };
 
-/**
- * An abstract type that represents a location in a source file.
- */
-struct AbstractPos
-{
-    uint32_t line = 0;
-    uint32_t column = 0;
-
-    /**
-     * An AbstractPos may be a "null object", representing an unknown position.
-     *
-     * Return true if this position is known.
-     */
-    inline operator bool() const { return line != 0; };
-
-    /**
-     * Return the contents of the source file.
-     */
-    virtual std::optional<std::string> getSource() const
-    { return std::nullopt; };
-
-    virtual void print(std::ostream & out) const = 0;
-
-    std::optional<LinesOfCode> getCodeLines() const;
-
-    virtual ~AbstractPos() = default;
-
-    inline auto operator<=>(const AbstractPos& rhs) const = default;
-};
-
-std::ostream & operator << (std::ostream & str, const AbstractPos & pos);
+struct Pos;
 
 void printCodeLines(std::ostream & out,
     const std::string & prefix,
-    const AbstractPos & errPos,
+    const Pos & errPos,
     const LinesOfCode & loc);
 
 struct Trace {
-    std::shared_ptr<AbstractPos> pos;
+    std::shared_ptr<Pos> pos;
     hintformat hint;
     bool frame;
 };
@@ -114,7 +84,7 @@ inline bool operator>=(const Trace& lhs, const Trace& rhs);
 struct ErrorInfo {
     Verbosity level;
     hintformat msg;
-    std::shared_ptr<AbstractPos> errPos;
+    std::shared_ptr<Pos> errPos;
     std::list<Trace> traces;
 
     Suggestions suggestions;
@@ -185,12 +155,12 @@ public:
     }
 
     template<typename... Args>
-    void addTrace(std::shared_ptr<AbstractPos> && e, std::string_view fs, const Args & ... args)
+    void addTrace(std::shared_ptr<Pos> && e, std::string_view fs, const Args & ... args)
     {
         addTrace(std::move(e), hintfmt(std::string(fs), args...));
     }
 
-    void addTrace(std::shared_ptr<AbstractPos> && e, hintformat hint, bool frame = false);
+    void addTrace(std::shared_ptr<Pos> && e, hintformat hint, bool frame = false);
 
     bool hasTrace() const { return !err.traces.empty(); }
 
diff --git a/src/libutil/input-accessor.hh b/src/libutil/input-accessor.hh
new file mode 100644
index 000000000..55b7c2f2f
--- /dev/null
+++ b/src/libutil/input-accessor.hh
@@ -0,0 +1,27 @@
+#pragma once
+///@file
+
+#include "source-accessor.hh"
+#include "ref.hh"
+#include "repair-flag.hh"
+
+namespace nix {
+
+MakeError(RestrictedPathError, Error);
+
+struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this<InputAccessor>
+{
+    std::optional<std::string> fingerprint;
+
+    /**
+     * Return the maximum last-modified time of the files in this
+     * tree, if available.
+     */
+    virtual std::optional<time_t> getLastModified()
+    {
+        return std::nullopt;
+    }
+
+};
+
+}
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 60b0865bf..183aee2dc 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -4,6 +4,8 @@
 #include "terminal.hh"
 #include "util.hh"
 #include "config.hh"
+#include "source-path.hh"
+#include "position.hh"
 
 #include <atomic>
 #include <nlohmann/json.hpp>
@@ -136,13 +138,13 @@ Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type,
     logger.startActivity(id, lvl, type, s, fields, parent);
 }
 
-void to_json(nlohmann::json & json, std::shared_ptr<AbstractPos> pos)
+void to_json(nlohmann::json & json, std::shared_ptr<Pos> pos)
 {
     if (pos) {
         json["line"] = pos->line;
         json["column"] = pos->column;
         std::ostringstream str;
-        pos->print(str);
+        pos->print(str, true);
         json["file"] = str.str();
     } else {
         json["line"] = nullptr;
diff --git a/src/libutil/position.cc b/src/libutil/position.cc
new file mode 100644
index 000000000..b39a5a1d4
--- /dev/null
+++ b/src/libutil/position.cc
@@ -0,0 +1,112 @@
+#include "position.hh"
+
+namespace nix {
+
+Pos::Pos(const Pos * other)
+{
+    if (!other) {
+        return;
+    }
+    line = other->line;
+    column = other->column;
+    origin = std::move(other->origin);
+}
+
+Pos::operator std::shared_ptr<Pos>() const
+{
+    return std::make_shared<Pos>(&*this);
+}
+
+bool Pos::operator<(const Pos &rhs) const
+{
+    return std::forward_as_tuple(line, column, origin)
+        < std::forward_as_tuple(rhs.line, rhs.column, rhs.origin);
+}
+
+std::optional<LinesOfCode> Pos::getCodeLines() const
+{
+    if (line == 0)
+        return std::nullopt;
+
+    if (auto source = getSource()) {
+
+        std::istringstream iss(*source);
+        // count the newlines.
+        int count = 0;
+        std::string curLine;
+        int pl = line - 1;
+
+        LinesOfCode loc;
+
+        do {
+            std::getline(iss, curLine);
+            ++count;
+            if (count < pl)
+                ;
+            else if (count == pl) {
+                loc.prevLineOfCode = curLine;
+            } else if (count == pl + 1) {
+                loc.errLineOfCode = curLine;
+            } else if (count == pl + 2) {
+                loc.nextLineOfCode = curLine;
+                break;
+            }
+
+            if (!iss.good())
+                break;
+        } while (true);
+
+        return loc;
+    }
+
+    return std::nullopt;
+}
+
+
+std::optional<std::string> Pos::getSource() const
+{
+    return std::visit(overloaded {
+        [](const std::monostate &) -> std::optional<std::string> {
+            return std::nullopt;
+        },
+        [](const Pos::Stdin & s) -> std::optional<std::string> {
+            // Get rid of the null terminators added by the parser.
+            return std::string(s.source->c_str());
+        },
+        [](const Pos::String & s) -> std::optional<std::string> {
+            // Get rid of the null terminators added by the parser.
+            return std::string(s.source->c_str());
+        },
+        [](const SourcePath & path) -> std::optional<std::string> {
+            try {
+                return path.readFile();
+            } catch (Error &) {
+                return std::nullopt;
+            }
+        }
+    }, origin);
+}
+
+void Pos::print(std::ostream & out, bool showOrigin) const
+{
+    if (showOrigin) {
+        std::visit(overloaded {
+            [&](const std::monostate &) { out << "«none»"; },
+            [&](const Pos::Stdin &) { out << "«stdin»"; },
+            [&](const Pos::String & s) { out << "«string»"; },
+            [&](const SourcePath & path) { out << path; }
+        }, origin);
+        out << ":";
+    }
+    out << line;
+    if (column > 0)
+        out << ":" << column;
+}
+
+std::ostream & operator<<(std::ostream & str, const Pos & pos)
+{
+    pos.print(str, true);
+    return str;
+}
+
+}
diff --git a/src/libutil/position.hh b/src/libutil/position.hh
new file mode 100644
index 000000000..a184997ed
--- /dev/null
+++ b/src/libutil/position.hh
@@ -0,0 +1,74 @@
+#pragma once
+/**
+ * @file
+ *
+ * @brief Pos and AbstractPos
+ */
+
+#include <cstdint>
+#include <string>
+
+#include "source-path.hh"
+
+namespace nix {
+
+/**
+ * A position and an origin for that position (like a source file).
+ */
+struct Pos
+{
+    uint32_t line = 0;
+    uint32_t column = 0;
+
+    struct Stdin {
+        ref<std::string> source;
+        bool operator==(const Stdin & rhs) const
+        { return *source == *rhs.source; }
+        bool operator!=(const Stdin & rhs) const
+        { return *source != *rhs.source; }
+        bool operator<(const Stdin & rhs) const
+        { return *source < *rhs.source; }
+    };
+    struct String {
+        ref<std::string> source;
+        bool operator==(const String & rhs) const
+        { return *source == *rhs.source; }
+        bool operator!=(const String & rhs) const
+        { return *source != *rhs.source; }
+        bool operator<(const String & rhs) const
+        { return *source < *rhs.source; }
+    };
+
+    typedef std::variant<std::monostate, Stdin, String, SourcePath> Origin;
+
+    Origin origin = std::monostate();
+
+    Pos() { }
+    Pos(uint32_t line, uint32_t column, Origin origin)
+        : line(line), column(column), origin(origin) { }
+    Pos(Pos & other) = default;
+    Pos(const Pos & other) = default;
+    Pos(Pos && other) = default;
+    Pos(const Pos * other);
+
+    explicit operator bool() const { return line > 0; }
+
+    operator std::shared_ptr<Pos>() const;
+
+    /**
+     * Return the contents of the source file.
+     */
+    std::optional<std::string> getSource() const;
+
+    void print(std::ostream & out, bool showOrigin) const;
+
+    std::optional<LinesOfCode> getCodeLines() const;
+
+    bool operator==(const Pos & rhs) const = default;
+    bool operator!=(const Pos & rhs) const = default;
+    bool operator<(const Pos & rhs) const;
+};
+
+std::ostream & operator<<(std::ostream & str, const Pos & pos);
+
+}
diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh
index af5f8304c..5d0c3696d 100644
--- a/src/libutil/ref.hh
+++ b/src/libutil/ref.hh
@@ -1,6 +1,7 @@
 #pragma once
 ///@file
 
+#include <compare>
 #include <memory>
 #include <exception>
 #include <stdexcept>
diff --git a/src/libstore/repair-flag.hh b/src/libutil/repair-flag.hh
similarity index 100%
rename from src/libstore/repair-flag.hh
rename to src/libutil/repair-flag.hh
diff --git a/src/libutil/source-path.cc b/src/libutil/source-path.cc
new file mode 100644
index 000000000..d85b0b7fe
--- /dev/null
+++ b/src/libutil/source-path.cc
@@ -0,0 +1,105 @@
+#include "source-path.hh"
+
+namespace nix {
+
+std::string_view SourcePath::baseName() const
+{ return path.baseName().value_or("source"); }
+
+SourcePath SourcePath::parent() const
+{
+    auto p = path.parent();
+    assert(p);
+    return {accessor, std::move(*p)};
+}
+
+std::string SourcePath::readFile() const
+{ return accessor->readFile(path); }
+
+bool SourcePath::pathExists() const
+{ return accessor->pathExists(path); }
+
+InputAccessor::Stat SourcePath::lstat() const
+{ return accessor->lstat(path); }
+
+std::optional<InputAccessor::Stat> SourcePath::maybeLstat() const
+{ return accessor->maybeLstat(path); }
+
+InputAccessor::DirEntries SourcePath::readDirectory() const
+{ return accessor->readDirectory(path); }
+
+std::string SourcePath::readLink() const
+{ return accessor->readLink(path); }
+
+void SourcePath::dumpPath(
+    Sink & sink,
+    PathFilter & filter) const
+{ return accessor->dumpPath(path, sink, filter); }
+
+std::optional<CanonPath> SourcePath::getPhysicalPath() const
+{ return accessor->getPhysicalPath(path); }
+
+std::string SourcePath::to_string() const
+{ return accessor->showPath(path); }
+
+SourcePath SourcePath::operator+(const CanonPath & x) const
+{ return {accessor, path + x}; }
+
+SourcePath SourcePath::operator+(std::string_view c) const
+{  return {accessor, path + c}; }
+
+bool SourcePath::operator==(const SourcePath & x) const
+{
+    return std::tie(*accessor, path) == std::tie(*x.accessor, x.path);
+}
+
+bool SourcePath::operator!=(const SourcePath & x) const
+{
+    return std::tie(*accessor, path) != std::tie(*x.accessor, x.path);
+}
+
+bool SourcePath::operator<(const SourcePath & x) const
+{
+    return std::tie(*accessor, path) < std::tie(*x.accessor, x.path);
+}
+
+SourcePath SourcePath::resolveSymlinks() const
+{
+    auto res = SourcePath(accessor);
+
+    int linksAllowed = 1024;
+
+    std::list<std::string> todo;
+    for (auto & c : path)
+        todo.push_back(std::string(c));
+
+    while (!todo.empty()) {
+        auto c = *todo.begin();
+        todo.pop_front();
+        if (c == "" || c == ".")
+            ;
+        else if (c == "..")
+            res.path.pop();
+        else {
+            res.path.push(c);
+            if (auto st = res.maybeLstat(); st && st->type == InputAccessor::tSymlink) {
+                if (!linksAllowed--)
+                    throw Error("infinite symlink recursion in path '%s'", path);
+                auto target = res.readLink();
+                res.path.pop();
+                if (hasPrefix(target, "/"))
+                    res.path = CanonPath::root;
+                todo.splice(todo.begin(), tokenizeString<std::list<std::string>>(target, "/"));
+            }
+        }
+    }
+
+    return res;
+}
+
+std::ostream & operator<<(std::ostream & str, const SourcePath & path)
+{
+    str << path.to_string();
+    return str;
+}
+
+}
diff --git a/src/libutil/source-path.hh b/src/libutil/source-path.hh
new file mode 100644
index 000000000..bf5625ca5
--- /dev/null
+++ b/src/libutil/source-path.hh
@@ -0,0 +1,114 @@
+#pragma once
+/**
+ * @file
+ *
+ * @brief SourcePath
+ */
+
+#include "ref.hh"
+#include "canon-path.hh"
+#include "input-accessor.hh"
+
+namespace nix {
+
+/**
+ * An abstraction for accessing source files during
+ * evaluation. Currently, it's just a wrapper around `CanonPath` that
+ * accesses files in the regular filesystem, but in the future it will
+ * support fetching files in other ways.
+ */
+struct SourcePath
+{
+    ref<InputAccessor> accessor;
+    CanonPath path;
+
+    SourcePath(ref<InputAccessor> accessor, CanonPath path = CanonPath::root)
+        : accessor(std::move(accessor))
+        , path(std::move(path))
+    { }
+
+    std::string_view baseName() const;
+
+    /**
+     * Construct the parent of this `SourcePath`. Aborts if `this`
+     * denotes the root.
+     */
+    SourcePath parent() const;
+
+    /**
+     * If this `SourcePath` denotes a regular file (not a symlink),
+     * return its contents; otherwise throw an error.
+     */
+    std::string readFile() const;
+
+    /**
+     * Return whether this `SourcePath` denotes a file (of any type)
+     * that exists
+    */
+    bool pathExists() const;
+
+    /**
+     * Return stats about this `SourcePath`, or throw an exception if
+     * it doesn't exist.
+     */
+    InputAccessor::Stat lstat() const;
+
+    /**
+     * Return stats about this `SourcePath`, or std::nullopt if it
+     * doesn't exist.
+     */
+    std::optional<InputAccessor::Stat> maybeLstat() const;
+
+    /**
+     * If this `SourcePath` denotes a directory (not a symlink),
+     * return its directory entries; otherwise throw an error.
+     */
+    InputAccessor::DirEntries readDirectory() const;
+
+    /**
+     * If this `SourcePath` denotes a symlink, return its target;
+     * otherwise throw an error.
+     */
+    std::string readLink() const;
+
+    /**
+     * Dump this `SourcePath` to `sink` as a NAR archive.
+     */
+    void dumpPath(
+        Sink & sink,
+        PathFilter & filter = defaultPathFilter) const;
+
+    /**
+     * Return the location of this path in the "real" filesystem, if
+     * it has a physical location.
+     */
+    std::optional<CanonPath> getPhysicalPath() const;
+
+    std::string to_string() const;
+
+    /**
+     * Append a `CanonPath` to this path.
+     */
+    SourcePath operator + (const CanonPath & x) const;
+
+    /**
+     * Append a single component `c` to this path. `c` must not
+     * contain a slash. A slash is implicitly added between this path
+     * and `c`.
+     */
+    SourcePath operator+(std::string_view c) const;
+    bool operator==(const SourcePath & x) const;
+    bool operator!=(const SourcePath & x) const;
+    bool operator<(const SourcePath & x) const;
+
+    /**
+     * Resolve any symlinks in this `SourcePath` (including its
+     * parents). The result is a `SourcePath` in which no element is a
+     * symlink.
+     */
+    SourcePath resolveSymlinks() const;
+};
+
+std::ostream & operator << (std::ostream & str, const SourcePath & path);
+
+}

From bbd0a959e17e988ef1ec2fadd1ab5bb66420fd6f Mon Sep 17 00:00:00 2001
From: Weijia Wang <9713184+wegank@users.noreply.github.com>
Date: Mon, 8 Jan 2024 20:37:42 +0100
Subject: [PATCH 320/421] Make lowdown optional

Co-authored-by: John Ericson <John.Ericson@Obsidian.Systems>
---
 configure.ac           | 16 +++++++++++++++-
 package.nix            |  5 +++++
 src/libcmd/markdown.cc |  6 ++++++
 3 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index b97e25bbd..929750932 100644
--- a/configure.ac
+++ b/configure.ac
@@ -374,7 +374,21 @@ PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9])
 
 
 # Look for lowdown library.
-PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"])
+AC_ARG_ENABLE([markdown], AS_HELP_STRING([--enable-markdown], [Enable Markdown rendering in the Nix binary (requires lowdown) [default=auto]]),
+  enable_markdown=$enableval, enable_markdown=auto)
+AS_CASE(["$enable_markdown"],
+  [yes | auto], [
+    PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [
+      CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"
+      have_lowdown=1
+      AC_DEFINE(HAVE_LOWDOWN, 1, [Whether lowdown is available and should be used for Markdown rendering.])
+    ], [
+      AS_IF([test "x$enable_markdown" == "xyes"], [AC_MSG_ERROR([--enable-markdown was specified, but lowdown was not found.])])
+    ])
+  ],
+  [no], [have_lowdown=],
+  [AC_MSG_ERROR([--enable-markdown must be one of: yes, no, auto])])
+AC_SUBST(HAVE_LOWDOWN, [$have_lowdown])
 
 
 # Look for libgit2.
diff --git a/package.nix b/package.nix
index dfebdb0e4..dd37809d0 100644
--- a/package.nix
+++ b/package.nix
@@ -68,6 +68,9 @@
 # Whether to build the regular manual
 , enableManual ? __forDefaults.canRunInstalled
 
+# Whether to enable Markdown rendering in the Nix binary.
+, enableMarkdown ? !stdenv.hostPlatform.isWindows
+
 # Whether to compile `rl-next.md`, the release notes for the next
 # not-yet-released version of Nix in the manul, from the individual
 # change log entries in the directory.
@@ -213,6 +216,7 @@ in {
     xz
   ] ++ lib.optionals (!stdenv.hostPlatform.isWindows) [
     editline
+  ] ++ lib.optionals enableMarkdown [
     lowdown
   ] ++ lib.optionals buildUnitTests [
     gtest
@@ -269,6 +273,7 @@ in {
     (lib.enableFeature doInstallCheck "functional-tests")
     (lib.enableFeature enableInternalAPIDocs "internal-api-docs")
     (lib.enableFeature enableManual "doc-gen")
+    (lib.enableFeature enableMarkdown "markdown")
     (lib.enableFeature installUnitTests "install-unit-tests")
   ] ++ lib.optionals (!forDevShell) [
     "--sysconfdir=/etc"
diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc
index 8b3bbc1b5..a4e3c5a77 100644
--- a/src/libcmd/markdown.cc
+++ b/src/libcmd/markdown.cc
@@ -4,12 +4,15 @@
 #include "terminal.hh"
 
 #include <sys/queue.h>
+#if HAVE_LOWDOWN
 #include <lowdown.h>
+#endif
 
 namespace nix {
 
 std::string renderMarkdownToTerminal(std::string_view markdown)
 {
+#if HAVE_LOWDOWN
     int windowWidth = getWindowSize().second;
 
     struct lowdown_opts opts {
@@ -48,6 +51,9 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
         throw Error("allocation error while rendering Markdown");
 
     return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI());
+#else
+    return std::string(markdown);
+#endif
 }
 
 }

From 29eb5ed1dc54ec45ab23b50ef259d2b370e8b1e8 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 8 Jan 2024 14:47:42 -0500
Subject: [PATCH 321/421] Fix Internal API docs

Because of source filtering, they were empty.

Fixes #9694
---
 package.nix | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/package.nix b/package.nix
index dfebdb0e4..aad022b32 100644
--- a/package.nix
+++ b/package.nix
@@ -164,6 +164,10 @@ in {
           ./doc/manual
         ] ++ lib.optionals enableInternalAPIDocs [
           ./doc/internal-api
+          # Source might not be compiled, but still must be available
+          # for Doxygen to gather comments.
+          ./src
+          ./tests/unit
         ] ++ lib.optionals buildUnitTests [
           ./tests/unit
         ] ++ lib.optionals doInstallCheck [

From 3d9e0c60e4cf135943d2c72a990ff2c0e3e311a7 Mon Sep 17 00:00:00 2001
From: DavHau <hsngrmpf+github@gmail.com>
Date: Tue, 9 Jan 2024 18:36:09 +0700
Subject: [PATCH 322/421] gitignore: add result-*

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index d9f9d949b..a47b195bb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -141,6 +141,7 @@ compile_commands.json
 nix-rust/target
 
 result
+result-*
 
 # IDE
 .vscode/

From 2cea88dbc8c277d7403e6dd65f482fd2eb931e52 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 2 Sep 2023 15:56:37 -0400
Subject: [PATCH 323/421] Improve build sytem support for readline instead of
 editline

Changes:

- CPP variable is now `USE_READLINE` not `READLINE`

- `configure.ac` supports with new CLI flag

- `package.nix` supports with new configuration option

- `flake.nix` CIs this (along with no markdown)

Remove old Ubuntu 16.04 stop-gap too, as that is now quite old.

Motivation:

- editline does not build for Windows, but readline *should*. (I am
  still working on this in Nixpkgs at this time, however. So there will
  be a follow-up Nix PR removing the windows-only skipping of the
  readline library once I am done.)

- Per
  https://salsa.debian.org/debian/nix/-/blob/master/debian/rules?ref_type=heads#L27
  and #2551, Debian builds Nix with readline. Now we better support and
  CI that build configuration.

This is picking up where #2551 left off, ensuring we test a few more
things not merely have CPP for them.

Co-authored-by: Weijia Wang <9713184+wegank@users.noreply.github.com>
---
 configure.ac       | 29 ++++++++++++++++++-----------
 flake.nix          |  9 +++++++++
 package.nix        | 12 +++++++++++-
 src/libcmd/repl.cc |  6 +++---
 4 files changed, 41 insertions(+), 15 deletions(-)

diff --git a/configure.ac b/configure.ac
index fdbb2629e..2594544ab 100644
--- a/configure.ac
+++ b/configure.ac
@@ -251,17 +251,25 @@ PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CX
 # Look for libcurl, a required dependency.
 PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"])
 
-# Look for editline, a required dependency.
+# Look for editline or readline, a required dependency.
 # The the libeditline.pc file was added only in libeditline >= 1.15.2,
 # see https://github.com/troglobit/editline/commit/0a8f2ef4203c3a4a4726b9dd1336869cd0da8607,
-# but e.g. Ubuntu 16.04 has an older version, so we fall back to searching for
-# editline.h when the pkg-config approach fails.
-PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"], [
-  AC_CHECK_HEADERS([editline.h], [true],
-    [AC_MSG_ERROR([Nix requires libeditline; it was found neither via pkg-config nor its normal header.])])
-  AC_SEARCH_LIBS([readline read_history], [editline], [],
-    [AC_MSG_ERROR([Nix requires libeditline; it was not found via pkg-config, but via its header, but required functions do not work. Maybe it is too old? >= 1.14 is required.])])
-])
+# Older versions are no longer supported.
+AC_ARG_WITH(
+  [readline-flavor],
+  AS_HELP_STRING([--with-readline-flavor],[Which library to use for nice line editting with the Nix language REPL" [default=editline]]),
+  [readline_flavor=$withval],
+  [readline_flavor=editline])
+AS_CASE(["$readline_flavor"],
+  [editline], [
+    readline_flavor_pc=libeditline
+  ],
+  [readline], [
+    readline_flavor_pc=readline
+    AC_DEFINE([USE_READLINE], [1], [Use readline instead of editline])
+  ],
+  [AC_MSG_ERROR([bad value "$readline_flavor" for --with-readline-flavor, must be one of: editline, readline])])
+PKG_CHECK_MODULES([EDITLINE], [$readline_flavor_pc], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"])
 
 # Look for libsodium.
 PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"])
@@ -387,8 +395,7 @@ AS_CASE(["$enable_markdown"],
     ])
   ],
   [no], [have_lowdown=],
-  [AC_MSG_ERROR([--enable-markdown must be one of: yes, no, auto])])
-AC_SUBST(HAVE_LOWDOWN, [$have_lowdown])
+  [AC_MSG_ERROR([bad value "$enable_markdown" for --enable-markdown, must be one of: yes, no, auto])])
 
 
 # Look for libgit2.
diff --git a/flake.nix b/flake.nix
index 32354a88f..c7aee7541 100644
--- a/flake.nix
+++ b/flake.nix
@@ -230,6 +230,15 @@
           }
         );
 
+        # Toggles some settings for better coverage. Windows needs these
+        # library combinations, and Debian build Nix with GNU readline too.
+        buildReadlineNoMarkdown = forAllSystems (system:
+          self.packages.${system}.nix.override {
+            enableMarkdown = false;
+            readlineFlavor = "readline";
+          }
+        );
+
         # Perl bindings for various platforms.
         perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings);
 
diff --git a/package.nix b/package.nix
index 727f5e646..d0b9fc3f3 100644
--- a/package.nix
+++ b/package.nix
@@ -13,6 +13,7 @@
 , changelog-d
 , curl
 , editline
+, readline
 , fileset
 , flex
 , git
@@ -71,6 +72,14 @@
 # Whether to enable Markdown rendering in the Nix binary.
 , enableMarkdown ? !stdenv.hostPlatform.isWindows
 
+# Which interactive line editor library to use for Nix's repl.
+#
+# Currently supported choices are:
+#
+# - editline (default)
+# - readline
+, readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline"
+
 # Whether to compile `rl-next.md`, the release notes for the next
 # not-yet-released version of Nix in the manul, from the individual
 # change log entries in the directory.
@@ -219,7 +228,7 @@ in {
     sqlite
     xz
   ] ++ lib.optionals (!stdenv.hostPlatform.isWindows) [
-    editline
+    ({ inherit readline editline; }.${readlineFlavor})
   ] ++ lib.optionals enableMarkdown [
     lowdown
   ] ++ lib.optionals buildUnitTests [
@@ -279,6 +288,7 @@ in {
     (lib.enableFeature enableManual "doc-gen")
     (lib.enableFeature enableMarkdown "markdown")
     (lib.enableFeature installUnitTests "install-unit-tests")
+    (lib.withFeatureAs true "readline-flavor" readlineFlavor)
   ] ++ lib.optionals (!forDevShell) [
     "--sysconfdir=/etc"
   ] ++ lib.optionals installUnitTests [
diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc
index dea91ba63..9c92f2b6e 100644
--- a/src/libcmd/repl.cc
+++ b/src/libcmd/repl.cc
@@ -5,7 +5,7 @@
 
 #include <setjmp.h>
 
-#ifdef READLINE
+#ifdef USE_READLINE
 #include <readline/history.h>
 #include <readline/readline.h>
 #else
@@ -249,14 +249,14 @@ void NixRepl::mainLoop()
     } catch (SysError & e) {
         logWarning(e.info());
     }
-#ifndef READLINE
+#ifndef USE_READLINE
     el_hist_size = 1000;
 #endif
     read_history(historyFile.c_str());
     auto oldRepl = curRepl;
     curRepl = this;
     Finally restoreRepl([&] { curRepl = oldRepl; });
-#ifndef READLINE
+#ifndef USE_READLINE
     rl_set_complete_func(completionCallback);
     rl_set_list_possib_func(listPossibleCallback);
 #endif

From 0c3ce237549d43de52e897f12e6d6c8ee59ac227 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 9 Jan 2024 17:31:40 -0500
Subject: [PATCH 324/421] Improve the build without GC

We don't just want to pass `--enable-gc=no`; we also want to make sure
boehmgc is not a dependency. Creating a nix-level configuration option
to do both, and then using that for the CI job, is more robust.
---
 flake.nix   |  4 +++-
 package.nix | 12 ++++++++++--
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/flake.nix b/flake.nix
index c7aee7541..49f214e72 100644
--- a/flake.nix
+++ b/flake.nix
@@ -220,7 +220,9 @@
         buildCross = forAllCrossSystems (crossSystem:
           lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}"));
 
-        buildNoGc = forAllSystems (system: self.packages.${system}.nix.overrideAttrs (a: { configureFlags = (a.configureFlags or []) ++ ["--enable-gc=no"];}));
+        buildNoGc = forAllSystems (system:
+          self.packages.${system}.nix.override { enableGC = false; }
+        );
 
         buildNoTests = forAllSystems (system:
           self.packages.${system}.nix.override {
diff --git a/package.nix b/package.nix
index d0b9fc3f3..71ee80e33 100644
--- a/package.nix
+++ b/package.nix
@@ -69,6 +69,14 @@
 # Whether to build the regular manual
 , enableManual ? __forDefaults.canRunInstalled
 
+# Whether to use garbage collection for the Nix language evaluator.
+#
+# If it is disabled, we just leak memory, but this is not as bad as it
+# sounds so long as evaluation just takes places within short-lived
+# processes. (When the process exits, the memory is reclaimed; it is
+# only leaked *within* the process.)
+, enableGC ? true
+
 # Whether to enable Markdown rendering in the Nix binary.
 , enableMarkdown ? !stdenv.hostPlatform.isWindows
 
@@ -245,9 +253,8 @@ in {
   ;
 
   propagatedBuildInputs = [
-    boehmgc
     nlohmann_json
-  ];
+  ] ++ lib.optional enableGC boehmgc;
 
   dontBuild = !attrs.doBuild;
   doCheck = attrs.doCheck;
@@ -286,6 +293,7 @@ in {
     (lib.enableFeature doInstallCheck "functional-tests")
     (lib.enableFeature enableInternalAPIDocs "internal-api-docs")
     (lib.enableFeature enableManual "doc-gen")
+    (lib.enableFeature enableGC "gc")
     (lib.enableFeature enableMarkdown "markdown")
     (lib.enableFeature installUnitTests "install-unit-tests")
     (lib.withFeatureAs true "readline-flavor" readlineFlavor)

From 57dc4fc878bc74dfb38cd9d435a85c560b43cebb Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 2 Sep 2023 16:21:44 -0400
Subject: [PATCH 325/421] Make more expressive `HOST_*` macro system

---
 mk/lib.mk | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/mk/lib.mk b/mk/lib.mk
index 3d503364f..a5a067e48 100644
--- a/mk/lib.mk
+++ b/mk/lib.mk
@@ -14,20 +14,34 @@ install-tests-groups :=
 
 ifdef HOST_OS
   HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS)))
+  ifeq ($(patsubst mingw%,,$(HOST_KERNEL)),)
+    HOST_MINGW = 1
+    HOST_WINDOWS = 1
+  endif
   ifeq ($(HOST_KERNEL), cygwin)
     HOST_CYGWIN = 1
+    HOST_WINDOWS = 1
+    HOST_UNIX = 1
   endif
   ifeq ($(patsubst darwin%,,$(HOST_KERNEL)),)
     HOST_DARWIN = 1
+    HOST_UNIX = 1
   endif
   ifeq ($(patsubst freebsd%,,$(HOST_KERNEL)),)
     HOST_FREEBSD = 1
+    HOST_UNIX = 1
+  endif
+  ifeq ($(patsubst netbsd%,,$(HOST_KERNEL)),)
+    HOST_NETBSD = 1
+    HOST_UNIX = 1
   endif
   ifeq ($(HOST_KERNEL), linux)
     HOST_LINUX = 1
+    HOST_UNIX = 1
   endif
   ifeq ($(patsubst solaris%,,$(HOST_KERNEL)),)
     HOST_SOLARIS = 1
+    HOST_UNIX = 1
   endif
 endif
 

From f9e5eb5f0a61555d24fe85b8edccf49f0b176152 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 10 Jan 2024 20:26:02 -0500
Subject: [PATCH 326/421] Make indentation in makesfiles consistent

Tab (as required) for rules, two spaces for `if`...`endif`.
---
 src/libexpr/local.mk      |  2 +-
 src/libstore/local.mk     | 12 ++++++------
 src/libutil/local.mk      |  2 +-
 tests/functional/local.mk | 10 +++++-----
 4 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index ed6bc761a..b60936a0e 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -18,7 +18,7 @@ libexpr_LIBS = libutil libstore libfetchers
 
 libexpr_LDFLAGS += -lboost_context -pthread
 ifdef HOST_LINUX
- libexpr_LDFLAGS += -ldl
+  libexpr_LDFLAGS += -ldl
 endif
 
 # The dependency on libgc must be propagated (i.e. meaning that
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index 675976314..f447e190d 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -16,15 +16,15 @@ endif
 $(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox)))
 
 ifeq ($(ENABLE_S3), 1)
-	libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-crt-cpp
+  libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-crt-cpp
 endif
 
 ifdef HOST_SOLARIS
-	libstore_LDFLAGS += -lsocket
+  libstore_LDFLAGS += -lsocket
 endif
 
 ifeq ($(HAVE_SECCOMP), 1)
-	libstore_LDFLAGS += $(LIBSECCOMP_LIBS)
+  libstore_LDFLAGS += $(LIBSECCOMP_LIBS)
 endif
 
 libstore_CXXFLAGS += \
@@ -48,9 +48,9 @@ $(d)/embedded-sandbox-shell.gen.hh: $(sandbox_shell)
 	$(trace-gen) hexdump -v -e '1/1 "0x%x," "\n"' < $< > $@.tmp
 	@mv $@.tmp $@
 else
-ifneq ($(sandbox_shell),)
-libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\""
-endif
+  ifneq ($(sandbox_shell),)
+    libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\""
+  endif
 endif
 
 $(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
diff --git a/src/libutil/local.mk b/src/libutil/local.mk
index 0fdebaf5c..6e3d6d052 100644
--- a/src/libutil/local.mk
+++ b/src/libutil/local.mk
@@ -17,5 +17,5 @@ $(foreach i, $(wildcard $(d)/signature/*.hh), \
 
 
 ifeq ($(HAVE_LIBCPUID), 1)
-	libutil_LDFLAGS += -lcpuid
+  libutil_LDFLAGS += -lcpuid
 endif
diff --git a/tests/functional/local.mk b/tests/functional/local.mk
index 192e275e3..25fcbcfe7 100644
--- a/tests/functional/local.mk
+++ b/tests/functional/local.mk
@@ -129,15 +129,15 @@ nix_tests = \
   impure-env.sh
 
 ifeq ($(HAVE_LIBCPUID), 1)
-	nix_tests += compute-levels.sh
+  nix_tests += compute-levels.sh
 endif
 
 ifeq ($(ENABLE_BUILD), yes)
-	nix_tests += test-libstoreconsumer.sh
+  nix_tests += test-libstoreconsumer.sh
 
-	ifeq ($(BUILD_SHARED_LIBS), 1)
-		nix_tests += plugins.sh
-	endif
+  ifeq ($(BUILD_SHARED_LIBS), 1)
+    nix_tests += plugins.sh
+  endif
 endif
 
 $(d)/test-libstoreconsumer.sh.test $(d)/test-libstoreconsumer.sh.test-debug: \

From 423484ad26850046c101affc9ff6ac4c36ccda06 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 9 Jan 2024 12:29:37 -0500
Subject: [PATCH 327/421] Only link with `-pthread` on Unix

We don't want this with MinGW.
---
 mk/libraries.mk                                 | 6 ++++++
 src/libcmd/local.mk                             | 2 +-
 src/libexpr/local.mk                            | 2 +-
 src/libfetchers/local.mk                        | 2 +-
 src/libstore/local.mk                           | 2 +-
 src/libutil/local.mk                            | 2 +-
 src/nix/local.mk                                | 2 +-
 tests/functional/test-libstoreconsumer/local.mk | 2 +-
 tests/unit/libexpr-support/local.mk             | 2 +-
 tests/unit/libstore-support/local.mk            | 2 +-
 tests/unit/libutil-support/local.mk             | 2 +-
 11 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/mk/libraries.mk b/mk/libraries.mk
index 1bc73d7f7..515a481f6 100644
--- a/mk/libraries.mk
+++ b/mk/libraries.mk
@@ -10,6 +10,12 @@ else
   endif
 endif
 
+ifdef HOST_UNIX
+  THREAD_LDFLAGS = -pthread
+else
+  THREAD_LDFLAGS =
+endif
+
 # Build a library with symbolic name $(1).  The library is defined by
 # various variables prefixed by ‘$(1)_’:
 #
diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk
index afd35af08..abb7459a7 100644
--- a/src/libcmd/local.mk
+++ b/src/libcmd/local.mk
@@ -8,7 +8,7 @@ libcmd_SOURCES := $(wildcard $(d)/*.cc)
 
 libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers
 
-libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread
+libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS)
 
 libcmd_LIBS = libstore libutil libexpr libmain libfetchers
 
diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index b60936a0e..0c3e36750 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -16,7 +16,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib
 
 libexpr_LIBS = libutil libstore libfetchers
 
-libexpr_LDFLAGS += -lboost_context -pthread
+libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS)
 ifdef HOST_LINUX
   libexpr_LDFLAGS += -ldl
 endif
diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk
index 266e7a211..e54db4937 100644
--- a/src/libfetchers/local.mk
+++ b/src/libfetchers/local.mk
@@ -8,6 +8,6 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc)
 
 libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
 
-libfetchers_LDFLAGS += -pthread $(LIBGIT2_LIBS) -larchive
+libfetchers_LDFLAGS += $(THREAD_LDFLAGS) $(LIBGIT2_LIBS) -larchive
 
 libfetchers_LIBS = libutil libstore
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index f447e190d..f86643849 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -8,7 +8,7 @@ libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc)
 
 libstore_LIBS = libutil
 
-libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) -pthread
+libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) $(THREAD_LDFLAGS)
 ifdef HOST_LINUX
  libstore_LDFLAGS += -ldl
 endif
diff --git a/src/libutil/local.mk b/src/libutil/local.mk
index 6e3d6d052..200026c1e 100644
--- a/src/libutil/local.mk
+++ b/src/libutil/local.mk
@@ -8,7 +8,7 @@ libutil_SOURCES := $(wildcard $(d)/*.cc $(d)/signature/*.cc)
 
 libutil_CXXFLAGS += -I src/libutil
 
-libutil_LDFLAGS += -pthread $(LIBCURL_LIBS) $(SODIUM_LIBS) $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
+libutil_LDFLAGS += $(THREAD_LDFLAGS) $(LIBCURL_LIBS) $(SODIUM_LIBS) $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
 
 $(foreach i, $(wildcard $(d)/args/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/args, 0644)))
diff --git a/src/nix/local.mk b/src/nix/local.mk
index a21aa705f..1d6f560d6 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -18,7 +18,7 @@ nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr
 
 nix_LIBS = libexpr libmain libfetchers libstore libutil libcmd
 
-nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS)
+nix_LDFLAGS = $(THREAD_LDFLAGS) $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS)
 
 $(foreach name, \
   nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \
diff --git a/tests/functional/test-libstoreconsumer/local.mk b/tests/functional/test-libstoreconsumer/local.mk
index edc140723..a1825c405 100644
--- a/tests/functional/test-libstoreconsumer/local.mk
+++ b/tests/functional/test-libstoreconsumer/local.mk
@@ -12,4 +12,4 @@ test-libstoreconsumer_CXXFLAGS += -I src/libutil -I src/libstore
 
 test-libstoreconsumer_LIBS = libstore libutil
 
-test-libstoreconsumer_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS)
+test-libstoreconsumer_LDFLAGS = $(THREAD_LDFLAGS) $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS)
diff --git a/tests/unit/libexpr-support/local.mk b/tests/unit/libexpr-support/local.mk
index 12a76206a..0501de33c 100644
--- a/tests/unit/libexpr-support/local.mk
+++ b/tests/unit/libexpr-support/local.mk
@@ -20,4 +20,4 @@ libexpr-test-support_LIBS = \
     libstore-test-support libutil-test-support \
     libexpr libstore libutil
 
-libexpr-test-support_LDFLAGS := -pthread -lrapidcheck
+libexpr-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck
diff --git a/tests/unit/libstore-support/local.mk b/tests/unit/libstore-support/local.mk
index ff075c96a..56dedd825 100644
--- a/tests/unit/libstore-support/local.mk
+++ b/tests/unit/libstore-support/local.mk
@@ -18,4 +18,4 @@ libstore-test-support_LIBS = \
     libutil-test-support \
     libstore libutil
 
-libstore-test-support_LDFLAGS := -pthread -lrapidcheck
+libstore-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck
diff --git a/tests/unit/libutil-support/local.mk b/tests/unit/libutil-support/local.mk
index 2ee2cdb6c..5f7835c9f 100644
--- a/tests/unit/libutil-support/local.mk
+++ b/tests/unit/libutil-support/local.mk
@@ -16,4 +16,4 @@ libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES)
 
 libutil-test-support_LIBS = libutil
 
-libutil-test-support_LDFLAGS := -pthread -lrapidcheck
+libutil-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck

From 813c113b9ecfab917e6d43ac5831a4f207fecaf0 Mon Sep 17 00:00:00 2001
From: DavHau <hsngrmpf+github@gmail.com>
Date: Fri, 29 Dec 2023 15:15:16 +0700
Subject: [PATCH 328/421] initialize test suite for git fetchers

solves #9388

This utilizes nixos vm tests to allow:
- writing tests for fetchTree and fetchGit involving actual networking.
- writing small independent test cases by automating local and remote repository setup per test case.

This adds:
  - a gitea module setting up a gitea server
  - a setup module that simplifies writing test cases by automating the repo setup.
  - a simple git http test case

Other improvements:
For all nixos tests, add capability of overriding the nix version to test against.
This should make it easier to prevent regressions. If a new test is added it can simply be ran against any older nix version without having to backport the test.
For example, for running the container tests against nix 2.12.0:
`nix build "$(nix eval --raw .#hydraJobs.tests.containers --impure --apply 't: (t.forNix "2.12.0").drvPath')^*" -L`
---
 tests/nixos/default.nix                     |  30 ++++--
 tests/nixos/fetch-git/default.nix           |  60 +++++++++++
 tests/nixos/fetch-git/testsupport/gitea.nix |  63 ++++++++++++
 tests/nixos/fetch-git/testsupport/setup.nix | 106 ++++++++++++++++++++
 4 files changed, 252 insertions(+), 7 deletions(-)
 create mode 100644 tests/nixos/fetch-git/default.nix
 create mode 100644 tests/nixos/fetch-git/testsupport/gitea.nix
 create mode 100644 tests/nixos/fetch-git/testsupport/setup.nix

diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix
index 4459aa664..1a42f886c 100644
--- a/tests/nixos/default.nix
+++ b/tests/nixos/default.nix
@@ -5,14 +5,28 @@ let
   nixos-lib = import (nixpkgs + "/nixos/lib") { };
 
   # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests
-  runNixOSTestFor = system: test: nixos-lib.runTest {
-    imports = [ test ];
-    hostPkgs = nixpkgsFor.${system}.native;
-    defaults = {
-      nixpkgs.pkgs = nixpkgsFor.${system}.native;
+  runNixOSTestFor = system: test:
+    (nixos-lib.runTest {
+      imports = [ test ];
+      hostPkgs = nixpkgsFor.${system}.native;
+      defaults = {
+        nixpkgs.pkgs = nixpkgsFor.${system}.native;
+        nix.checkAllErrors = false;
+      };
+      _module.args.nixpkgs = nixpkgs;
+      _module.args.system = system;
+    })
+    // {
+      # allow running tests against older nix versions via `nix eval --apply`
+      # Example:
+      #   nix build "$(nix eval --raw --impure .#hydraJobs.tests.fetch-git --apply 't: (t.forNix "2.19.2").drvPath')^*"
+      forNix = nixVersion: runNixOSTestFor system {
+        imports = [test];
+        defaults.nixpkgs.overlays = [(curr: prev: {
+          nix = (builtins.getFlake "nix/${nixVersion}").packages.${system}.nix;
+        })];
+      };
     };
-    _module.args.nixpkgs = nixpkgs;
-  };
 
 in
 
@@ -40,4 +54,6 @@ in
   setuid = lib.genAttrs
     ["i686-linux" "x86_64-linux"]
     (system: runNixOSTestFor system ./setuid.nix);
+
+  fetch-git = runNixOSTestFor "x86_64-linux" ./fetch-git;
 }
diff --git a/tests/nixos/fetch-git/default.nix b/tests/nixos/fetch-git/default.nix
new file mode 100644
index 000000000..abeefb0e3
--- /dev/null
+++ b/tests/nixos/fetch-git/default.nix
@@ -0,0 +1,60 @@
+{ lib, config, ... }:
+{
+  name = "fetch-git";
+
+  imports = [
+    ./testsupport/gitea.nix
+  ];
+
+  /*
+  Test cases
+  The following is set up automatically for each test case:
+    - a repo with the {name} is created on the gitea server
+    - a repo with the {name} is created on the client
+    - the client repo is configured to push to the server repo
+  Python variables:
+    - repo.path: the path to the directory of the client repo
+    - repo.git: the git command with the client repo as the working directory
+    - repo.remote: the url to the server repo
+  */
+  testCases = [
+    {
+      name = "simple-http";
+      description = "can fetch a git repo via http";
+      script = ''
+        # add a file to the repo
+        client.succeed(f"""
+          echo chiang-mai > {repo.path}/thailand \
+          && {repo.git} add thailand \
+          && {repo.git} commit -m 'commit1'
+        """)
+
+        # memoize the revision
+        rev1 = client.succeed(f"""
+          {repo.git} rev-parse HEAD
+        """).strip()
+
+        # push to the server
+        client.succeed(f"""
+          {repo.git} push origin main
+        """)
+
+        # fetch the repo via nix
+        fetched1 = client.succeed(f"""
+          nix eval --impure --raw --expr "(builtins.fetchGit {repo.remote}).outPath"
+        """)
+
+        # check if the committed file is there
+        client.succeed(f"""
+          test -f {fetched1}/thailand
+        """)
+
+        # check if the revision is the same
+        rev1_fetched = client.succeed(f"""
+          nix eval --impure --raw --expr "(builtins.fetchGit {repo.remote}).rev"
+        """).strip()
+        assert rev1 == rev1_fetched
+      '';
+    }
+  ];
+}
diff --git a/tests/nixos/fetch-git/testsupport/gitea.nix b/tests/nixos/fetch-git/testsupport/gitea.nix
new file mode 100644
index 000000000..d2bd622e4
--- /dev/null
+++ b/tests/nixos/fetch-git/testsupport/gitea.nix
@@ -0,0 +1,63 @@
+{ lib, nixpkgs, system, ... }: {
+  imports = [
+    ../testsupport/setup.nix
+  ];
+  nodes = {
+    gitea = { pkgs, ... }: {
+      services.gitea.enable = true;
+      services.gitea.settings.service.DISABLE_REGISTRATION = true;
+      services.gitea.settings.log.LEVEL = "Info";
+      services.gitea.settings.database.LOG_SQL = false;
+      networking.firewall.allowedTCPPorts = [ 3000 ];
+      environment.systemPackages = [ pkgs.gitea ];
+
+      # TODO: remove this after updating to nixos-23.11
+      nixpkgs.pkgs = lib.mkForce (import nixpkgs {
+        inherit system;
+        config.permittedInsecurePackages = [
+          "gitea-1.19.4"
+        ];
+      });
+    };
+    client = { pkgs, ... }: {
+      environment.systemPackages = [ pkgs.git ];
+    };
+  };
+  defaults = { pkgs, ... }: {
+    environment.systemPackages = [ pkgs.jq ];
+  };
+
+  setupScript = ''
+    import shlex
+
+    gitea.wait_for_unit("gitea.service")
+
+    gitea_admin = "test"
+    gitea_admin_password = "test123test"
+
+    gitea.succeed(f"""
+      gitea --version >&2
+      su -l gitea -c 'GITEA_WORK_DIR=/var/lib/gitea gitea admin user create \
+        --username {gitea_admin} --password {gitea_admin_password} --email test@client'
+    """)
+
+    client.wait_for_unit("multi-user.target")
+    gitea.wait_for_open_port(3000)
+
+    gitea_admin_token = gitea.succeed(f"""
+      curl --fail -X POST http://{gitea_admin}:{gitea_admin_password}@gitea:3000/api/v1/users/test/tokens \
+        -H 'Accept: application/json' -H 'Content-Type: application/json' \
+        -d {shlex.quote( '{"name":"token", "scopes":["all"]}' )} \
+        | jq -r '.sha1'
+    """).strip()
+
+    client.succeed(f"""
+      echo "http://{gitea_admin}:{gitea_admin_password}@gitea:3000" >~/.git-credentials-admin
+      git config --global credential.helper 'store --file ~/.git-credentials-admin'
+      git config --global user.email "test@client"
+      git config --global user.name "Test User"
+      git config --global gc.autodetach 0
+      git config --global gc.auto 0
+    """)
+  '';
+}
diff --git a/tests/nixos/fetch-git/testsupport/setup.nix b/tests/nixos/fetch-git/testsupport/setup.nix
new file mode 100644
index 000000000..f2fbd737d
--- /dev/null
+++ b/tests/nixos/fetch-git/testsupport/setup.nix
@@ -0,0 +1,106 @@
+{ lib, config, extendModules, ... }:
+let
+  inherit (lib)
+    mkOption
+    types
+    ;
+
+  indent = lib.replaceStrings ["\n"] ["\n    "];
+
+  execTestCase = testCase: ''
+
+    ### TEST ${testCase.name}: ${testCase.description} ###
+
+    with subtest("${testCase.description}"):
+        repo = Repo("${testCase.name}")
+        ${indent testCase.script}
+  '';
+in
+{
+
+  options = {
+    setupScript = mkOption {
+      type = types.lines;
+      description = ''
+        Python code that runs before the main test.
+
+        Variables defined by this code will be available in the test.
+      '';
+      default = "";
+    };
+    testCases = mkOption {
+      description = ''
+        The test cases. See `testScript`.
+      '';
+      type = types.listOf (types.submodule {
+        options.name = mkOption {
+          type = types.str;
+          description = ''
+            The name of the test case.
+
+            A repository with that name will be set up on the gitea server and locally.
+
+            This name can also be used to execute only a single test case via:
+              `nix build .#hydraJobs.fetch-git.{test-case-name}`
+          '';
+        };
+        options.description = mkOption {
+          type = types.str;
+          description = ''
+            A description of the test case.
+          '';
+        };
+        options.script = mkOption {
+          type = types.lines;
+          description = ''
+            Python code that runs the test.
+
+            Variables defined by `setupScript` will be available here.
+          '';
+        };
+      });
+    };
+  };
+
+  config = {
+    nodes.client = {
+      environment.variables = {
+        _NIX_FORCE_HTTP = "1";
+      };
+      nix.settings.experimental-features = ["nix-command" "flakes"];
+    };
+    setupScript = ''
+      class Repo:
+        """
+        A class to create a git repository on the gitea server and locally.
+        """
+        def __init__(self, name):
+          self.name = name
+          self.path = "/tmp/repos/" + name
+          self.remote = "http://gitea:3000/test/" + name
+          self.git = f"git -C {self.path}"
+          self.create()
+
+        def create(self):
+          gitea.succeed(f"""
+            curl --fail -X POST http://{gitea_admin}:{gitea_admin_password}@gitea:3000/api/v1/user/repos \
+              -H 'Accept: application/json' -H 'Content-Type: application/json' \
+              -d {shlex.quote( f'{{"name":"{self.name}", "default_branch": "main"}}' )}
+          """)
+          client.succeed(f"""
+            mkdir -p {self.path} \
+            && git init -b main {self.path} \
+            && {self.git} remote add origin {self.remote}
+          """)
+    '';
+    testScript = ''
+      start_all();
+
+      ${config.setupScript}
+
+      ### SETUP COMPLETE ###
+
+      ${lib.concatStringsSep "\n" (map execTestCase config.testCases)}
+    '';
+  };
+}

From 0f95330fde6ebad95b5a50be3aacb0a1d11363af Mon Sep 17 00:00:00 2001
From: DavHau <hsngrmpf+github@gmail.com>
Date: Thu, 11 Jan 2024 14:41:35 +0700
Subject: [PATCH 329/421] fetchGit: add simple test for ssh fetching

Also move tests to separate files which are auto-imported. This should allow people adding tests concurrently without introducing merge conflicts
---
 tests/nixos/fetch-git/default.nix             | 52 +++++--------------
 .../test-cases/http-simple/default.nix        | 37 +++++++++++++
 .../test-cases/ssh-simple/default.nix         | 41 +++++++++++++++
 tests/nixos/fetch-git/testsupport/gitea.nix   | 41 ++++++++++++++-
 tests/nixos/fetch-git/testsupport/setup.nix   | 10 +++-
 5 files changed, 138 insertions(+), 43 deletions(-)
 create mode 100644 tests/nixos/fetch-git/test-cases/http-simple/default.nix
 create mode 100644 tests/nixos/fetch-git/test-cases/ssh-simple/default.nix

diff --git a/tests/nixos/fetch-git/default.nix b/tests/nixos/fetch-git/default.nix
index abeefb0e3..254fecaaf 100644
--- a/tests/nixos/fetch-git/default.nix
+++ b/tests/nixos/fetch-git/default.nix
@@ -8,53 +8,25 @@
 
   /*
   Test cases
+
+  Test cases are automatically imported from ./test-cases/{name}
+
   The following is set up automatically for each test case:
     - a repo with the {name} is created on the gitea server
     - a repo with the {name} is created on the client
     - the client repo is configured to push to the server repo
+
   Python variables:
     - repo.path: the path to the directory of the client repo
     - repo.git: the git command with the client repo as the working directory
     - repo.remote: the url to the server repo
   */
-  testCases = [
-    {
-      name = "simple-http";
-      description = "can fetch a git repo via http";
-      script = ''
-        # add a file to the repo
-        client.succeed(f"""
-          echo chiang-mai > {repo.path}/thailand \
-          && {repo.git} add thailand \
-          && {repo.git} commit -m 'commit1'
-        """)
-
-        # memoize the revision
-        rev1 = client.succeed(f"""
-          {repo.git} rev-parse HEAD
-        """).strip()
-
-        # push to the server
-        client.succeed(f"""
-          {repo.git} push origin main
-        """)
-
-        # fetch the repo via nix
-        fetched1 = client.succeed(f"""
-          nix eval --impure --raw --expr "(builtins.fetchGit {repo.remote}).outPath"
-        """)
-
-        # check if the committed file is there
-        client.succeed(f"""
-          test -f {fetched1}/thailand
-        """)
-
-        # check if the revision is the same
-        rev1_fetched = client.succeed(f"""
-          nix eval --impure --raw --expr "(builtins.fetchGit {repo.remote}).rev"
-        """).strip()
-        assert rev1 == rev1_fetched
-      '';
-    }
-  ];
+  testCases =
+    map
+    (testCaseName: {...}: {
+      imports = ["${./test-cases}/${testCaseName}"];
+      # ensures tests are named like their directories they are defined in
+      name = testCaseName;
+    })
+    (lib.attrNames (builtins.readDir ./test-cases));
 }
diff --git a/tests/nixos/fetch-git/test-cases/http-simple/default.nix b/tests/nixos/fetch-git/test-cases/http-simple/default.nix
new file mode 100644
index 000000000..1bd5bbba2
--- /dev/null
+++ b/tests/nixos/fetch-git/test-cases/http-simple/default.nix
@@ -0,0 +1,37 @@
+{
+  description = "can fetch a git repo via http";
+  script = ''
+    # add a file to the repo
+    client.succeed(f"""
+      echo chiang-mai > {repo.path}/thailand \
+      && {repo.git} add thailand \
+      && {repo.git} commit -m 'commit1'
+    """)
+
+    # memoize the revision
+    rev1 = client.succeed(f"""
+      {repo.git} rev-parse HEAD
+    """).strip()
+
+    # push to the server
+    client.succeed(f"""
+      {repo.git} push origin main
+    """)
+
+    # fetch the repo via nix
+    fetched1 = client.succeed(f"""
+      nix eval --impure --raw --expr "(builtins.fetchGit {repo.remote}).outPath"
+    """)
+
+    # check if the committed file is there
+    client.succeed(f"""
+      test -f {fetched1}/thailand
+    """)
+
+    # check if the revision is the same
+    rev1_fetched = client.succeed(f"""
+      nix eval --impure --raw --expr "(builtins.fetchGit {repo.remote}).rev"
+    """).strip()
+    assert rev1 == rev1_fetched, f"rev1: {rev1} != rev1_fetched: {rev1_fetched}"
+  '';
+}
diff --git a/tests/nixos/fetch-git/test-cases/ssh-simple/default.nix b/tests/nixos/fetch-git/test-cases/ssh-simple/default.nix
new file mode 100644
index 000000000..0e4494ae0
--- /dev/null
+++ b/tests/nixos/fetch-git/test-cases/ssh-simple/default.nix
@@ -0,0 +1,41 @@
+{
+  description = "can fetch a git repo via ssh";
+  script = ''
+    # add a file to the repo
+    client.succeed(f"""
+      echo chiang-mai > {repo.path}/thailand \
+      && {repo.git} add thailand \
+      && {repo.git} commit -m 'commit1'
+    """)
+
+    # memoize the revision
+    rev1 = client.succeed(f"""
+      {repo.git} rev-parse HEAD
+    """).strip()
+
+    # push to the server
+    client.succeed(f"""
+      {repo.git} push origin-ssh main
+    """)
+
+    # fetch the repo via nix
+    fetched1 = client.succeed(f"""
+      nix eval --impure --raw --expr '
+        (builtins.fetchGit "{repo.remote_ssh}").outPath
+      '
+    """)
+
+    # check if the committed file is there
+    client.succeed(f"""
+      test -f {fetched1}/thailand
+    """)
+
+    # check if the revision is the same
+    rev1_fetched = client.succeed(f"""
+      nix eval --impure --raw --expr '
+        (builtins.fetchGit "{repo.remote_ssh}").rev
+      '
+    """).strip()
+    assert rev1 == rev1_fetched, f"rev1: {rev1} != rev1_fetched: {rev1_fetched}"
+  '';
+}
diff --git a/tests/nixos/fetch-git/testsupport/gitea.nix b/tests/nixos/fetch-git/testsupport/gitea.nix
index d2bd622e4..2ea23961e 100644
--- a/tests/nixos/fetch-git/testsupport/gitea.nix
+++ b/tests/nixos/fetch-git/testsupport/gitea.nix
@@ -1,4 +1,18 @@
-{ lib, nixpkgs, system, ... }: {
+{ lib, nixpkgs, system, pkgs, ... }: let
+  clientPrivateKey = pkgs.writeText "id_ed25519" ''
+    -----BEGIN OPENSSH PRIVATE KEY-----
+    b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+    QyNTUxOQAAACBbeWvHh/AWGWI6EIc1xlSihyXtacNQ9KeztlW/VUy8wQAAAJAwVQ5VMFUO
+    VQAAAAtzc2gtZWQyNTUxOQAAACBbeWvHh/AWGWI6EIc1xlSihyXtacNQ9KeztlW/VUy8wQ
+    AAAEB7lbfkkdkJoE+4TKHPdPQWBKLSx+J54Eg8DaTr+3KoSlt5a8eH8BYZYjoQhzXGVKKH
+    Je1pw1D0p7O2Vb9VTLzBAAAACGJmb0BtaW5pAQIDBAU=
+    -----END OPENSSH PRIVATE KEY-----
+  '';
+
+  clientPublicKey =
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFt5a8eH8BYZYjoQhzXGVKKHJe1pw1D0p7O2Vb9VTLzB";
+
+in {
   imports = [
     ../testsupport/setup.nix
   ];
@@ -8,8 +22,11 @@
       services.gitea.settings.service.DISABLE_REGISTRATION = true;
       services.gitea.settings.log.LEVEL = "Info";
       services.gitea.settings.database.LOG_SQL = false;
+      services.openssh.enable = true;
       networking.firewall.allowedTCPPorts = [ 3000 ];
-      environment.systemPackages = [ pkgs.gitea ];
+      environment.systemPackages = [ pkgs.git pkgs.gitea ];
+
+      users.users.root.openssh.authorizedKeys.keys = [clientPublicKey];
 
       # TODO: remove this after updating to nixos-23.11
       nixpkgs.pkgs = lib.mkForce (import nixpkgs {
@@ -59,5 +76,25 @@
       git config --global gc.autodetach 0
       git config --global gc.auto 0
     """)
+
+    # add client's private key to ~/.ssh
+    client.succeed("""
+      mkdir -p ~/.ssh
+      chmod 700 ~/.ssh
+      cat ${clientPrivateKey} >~/.ssh/id_ed25519
+      chmod 600 ~/.ssh/id_ed25519
+    """)
+
+    client.succeed("""
+      echo "Host gitea" >>~/.ssh/config
+      echo "  StrictHostKeyChecking no" >>~/.ssh/config
+      echo "  UserKnownHostsFile /dev/null" >>~/.ssh/config
+      echo "  User root" >>~/.ssh/config
+    """)
+
+    # ensure ssh from client to gitea works
+    client.succeed("""
+      ssh root@gitea true
+    """)
   '';
 }
diff --git a/tests/nixos/fetch-git/testsupport/setup.nix b/tests/nixos/fetch-git/testsupport/setup.nix
index f2fbd737d..2f74f51f8 100644
--- a/tests/nixos/fetch-git/testsupport/setup.nix
+++ b/tests/nixos/fetch-git/testsupport/setup.nix
@@ -78,19 +78,27 @@ in
           self.name = name
           self.path = "/tmp/repos/" + name
           self.remote = "http://gitea:3000/test/" + name
+          self.remote_ssh = "ssh://gitea/root/" + name
           self.git = f"git -C {self.path}"
           self.create()
 
         def create(self):
+          # create ssh remote repo
+          gitea.succeed(f"""
+            git init --bare -b main /root/{self.name}
+          """)
+          # create http remote repo
           gitea.succeed(f"""
             curl --fail -X POST http://{gitea_admin}:{gitea_admin_password}@gitea:3000/api/v1/user/repos \
               -H 'Accept: application/json' -H 'Content-Type: application/json' \
               -d {shlex.quote( f'{{"name":"{self.name}", "default_branch": "main"}}' )}
           """)
+          # setup git remotes on client
           client.succeed(f"""
             mkdir -p {self.path} \
             && git init -b main {self.path} \
-            && {self.git} remote add origin {self.remote}
+            && {self.git} remote add origin {self.remote} \
+            && {self.git} remote add origin-ssh root@gitea:{self.name}
           """)
     '';
     testScript = ''

From a923444a9462cd2fabcd816fa2e9cb54c485f13f Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 11 Jan 2024 13:55:28 -0500
Subject: [PATCH 330/421] packages.nix: Fix `installUnitTests` condition

The intent was we install the tests when we can *not* run them. Instead,
we were installing them when we can.
---
 package.nix | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/package.nix b/package.nix
index 71ee80e33..37410dc2f 100644
--- a/package.nix
+++ b/package.nix
@@ -100,7 +100,7 @@
 # Whether to install unit tests. This is useful when cross compiling
 # since we cannot run them natively during the build, but can do so
 # later.
-, installUnitTests ? __forDefaults.canRunInstalled
+, installUnitTests ? doBuild && !__forDefaults.canExecuteHost
 
 # For running the functional tests against a pre-built Nix. Probably
 # want to use in conjunction with `doBuild = false;`.
@@ -113,7 +113,8 @@
 # Not a real argument, just the only way to approximate let-binding some
 # stuff for argument defaults.
 , __forDefaults ? {
-    canRunInstalled = doBuild && stdenv.buildPlatform.canExecute stdenv.hostPlatform;
+    canExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
+    canRunInstalled = doBuild && __forDefaults.canExecuteHost;
   }
 }:
 

From c9125603a535f82cc9a53f47533f0a3d174e7008 Mon Sep 17 00:00:00 2001
From: Rebecca Turner <rbt@sent.as>
Date: Mon, 18 Dec 2023 10:34:19 -0800
Subject: [PATCH 331/421] Unindent `print.hh` declarations

---
 src/libexpr/print.hh | 82 +++++++++++++++++++++++---------------------
 1 file changed, 42 insertions(+), 40 deletions(-)

diff --git a/src/libexpr/print.hh b/src/libexpr/print.hh
index 3b72ae201..abf830864 100644
--- a/src/libexpr/print.hh
+++ b/src/libexpr/print.hh
@@ -10,45 +10,47 @@
 #include <iostream>
 
 namespace nix {
-    /**
-     * Print a string as a Nix string literal.
-     *
-     * Quotes and fairly minimal escaping are added.
-     *
-     * @param s The logical string
-     */
-    std::ostream & printLiteralString(std::ostream & o, std::string_view s);
-    inline std::ostream & printLiteralString(std::ostream & o, const char * s) {
-        return printLiteralString(o, std::string_view(s));
-    }
-    inline std::ostream & printLiteralString(std::ostream & o, const std::string & s) {
-        return printLiteralString(o, std::string_view(s));
-    }
 
-    /** Print `true` or `false`. */
-    std::ostream & printLiteralBool(std::ostream & o, bool b);
-
-    /**
-     * Print a string as an attribute name in the Nix expression language syntax.
-     *
-     * Prints a quoted string if necessary.
-     */
-    std::ostream & printAttributeName(std::ostream & o, std::string_view s);
-
-    /**
-     * Returns `true' is a string is a reserved keyword which requires quotation
-     * when printing attribute set field names.
-     */
-    bool isReservedKeyword(const std::string_view str);
-
-    /**
-     * Print a string as an identifier in the Nix expression language syntax.
-     *
-     * FIXME: "identifier" is ambiguous. Identifiers do not have a single
-     *        textual representation. They can be used in variable references,
-     *        let bindings, left-hand sides or attribute names in a select
-     *        expression, or something else entirely, like JSON. Use one of the
-     *        `print*` functions instead.
-     */
-    std::ostream & printIdentifier(std::ostream & o, std::string_view s);
+/**
+ * Print a string as a Nix string literal.
+ *
+ * Quotes and fairly minimal escaping are added.
+ *
+ * @param s The logical string
+ */
+std::ostream & printLiteralString(std::ostream & o, std::string_view s);
+inline std::ostream & printLiteralString(std::ostream & o, const char * s) {
+    return printLiteralString(o, std::string_view(s));
+}
+inline std::ostream & printLiteralString(std::ostream & o, const std::string & s) {
+    return printLiteralString(o, std::string_view(s));
+}
+
+/** Print `true` or `false`. */
+std::ostream & printLiteralBool(std::ostream & o, bool b);
+
+/**
+ * Print a string as an attribute name in the Nix expression language syntax.
+ *
+ * Prints a quoted string if necessary.
+ */
+std::ostream & printAttributeName(std::ostream & o, std::string_view s);
+
+/**
+ * Returns `true' is a string is a reserved keyword which requires quotation
+ * when printing attribute set field names.
+ */
+bool isReservedKeyword(const std::string_view str);
+
+/**
+ * Print a string as an identifier in the Nix expression language syntax.
+ *
+ * FIXME: "identifier" is ambiguous. Identifiers do not have a single
+ *        textual representation. They can be used in variable references,
+ *        let bindings, left-hand sides or attribute names in a select
+ *        expression, or something else entirely, like JSON. Use one of the
+ *        `print*` functions instead.
+ */
+std::ostream & printIdentifier(std::ostream & o, std::string_view s);
+
 }

From 0fa08b451682fb3311fe58112ff05c4fe5bee3a4 Mon Sep 17 00:00:00 2001
From: Rebecca Turner <rbt@sent.as>
Date: Tue, 12 Dec 2023 13:57:36 -0800
Subject: [PATCH 332/421] Unify and refactor value printing

Previously, there were two mostly-identical value printers -- one in
`libexpr/eval.cc` (which didn't force values) and one in
`libcmd/repl.cc` (which did force values and also printed ANSI color
codes).

This PR unifies both of these printers into `print.cc` and provides a
`PrintOptions` struct for controlling the output, which allows for
toggling whether values are forced, whether repeated values are tracked,
and whether ANSI color codes are displayed.

Additionally, `PrintOptions` allows tuning the maximum number of
attributes, list items, and bytes in a string that will be displayed;
this makes it ideal for contexts where printing too much output (e.g.
all of Nixpkgs) is distracting. (As requested by @roberth in
https://github.com/NixOS/nix/pull/9554#issuecomment-1845095735)

Please read the tests for example output.

Future work:
- It would be nice to provide this function as a builtin, perhaps
  `builtins.toStringDebug` -- a printing function that never fails would
  be useful when debugging Nix code.
- It would be nice to support customizing `PrintOptions` members on the
  command line, e.g. `--option to-string-max-attrs 1000`.
---
 src/libcmd/repl.cc                            | 158 +----
 src/libexpr/eval.cc                           | 126 +---
 src/libexpr/eval.hh                           |   4 +-
 src/libexpr/print-options.hh                  |  52 ++
 src/libexpr/print.cc                          | 416 +++++++++++-
 src/libexpr/print.hh                          |   6 +
 src/libexpr/value.hh                          |  17 +-
 src/libutil/english.cc                        |  18 +
 src/libutil/english.hh                        |  18 +
 src/nix-env/user-env.cc                       |   5 +-
 src/nix-instantiate/nix-instantiate.cc        |   2 +-
 tests/functional/lang/eval-okay-print.err.exp |   2 +-
 tests/functional/lang/eval-okay-print.exp     |   2 +-
 .../lang/eval-okay-repeated-empty-attrs.exp   |   1 +
 .../lang/eval-okay-repeated-empty-attrs.nix   |   2 +
 .../lang/eval-okay-repeated-empty-list.exp    |   1 +
 .../lang/eval-okay-repeated-empty-list.nix    |   1 +
 tests/unit/libexpr/value/print.cc             | 621 +++++++++++++++++-
 18 files changed, 1174 insertions(+), 278 deletions(-)
 create mode 100644 src/libexpr/print-options.hh
 create mode 100644 src/libutil/english.cc
 create mode 100644 src/libutil/english.hh
 create mode 100644 tests/functional/lang/eval-okay-repeated-empty-attrs.exp
 create mode 100644 tests/functional/lang/eval-okay-repeated-empty-attrs.nix
 create mode 100644 tests/functional/lang/eval-okay-repeated-empty-list.exp
 create mode 100644 tests/functional/lang/eval-okay-repeated-empty-list.nix

diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc
index 7a1df74ef..72e3559df 100644
--- a/src/libcmd/repl.cc
+++ b/src/libcmd/repl.cc
@@ -93,9 +93,17 @@ struct NixRepl
     void evalString(std::string s, Value & v);
     void loadDebugTraceEnv(DebugTrace & dt);
 
-    typedef std::set<Value *> ValuesSeen;
-    std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth);
-    std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen);
+    void printValue(std::ostream & str,
+                              Value & v,
+                              unsigned int maxDepth = std::numeric_limits<unsigned int>::max())
+    {
+        ::nix::printValue(*state, str, v, PrintOptions {
+            .ansiColors = true,
+            .force = true,
+            .derivationPaths = true,
+            .maxDepth = maxDepth
+        });
+    }
 };
 
 std::string removeWhitespace(std::string s)
@@ -708,7 +716,8 @@ bool NixRepl::processLine(std::string line)
     else if (command == ":p" || command == ":print") {
         Value v;
         evalString(arg, v);
-        printValue(std::cout, v, 1000000000) << std::endl;
+        printValue(std::cout, v);
+        std::cout << std::endl;
     }
 
     else if (command == ":q" || command == ":quit") {
@@ -770,7 +779,8 @@ bool NixRepl::processLine(std::string line)
         } else {
             Value v;
             evalString(line, v);
-            printValue(std::cout, v, 1) << std::endl;
+            printValue(std::cout, v, 1);
+            std::cout << std::endl;
         }
     }
 
@@ -892,144 +902,6 @@ void NixRepl::evalString(std::string s, Value & v)
 }
 
 
-std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth)
-{
-    ValuesSeen seen;
-    return printValue(str, v, maxDepth, seen);
-}
-
-
-
-
-// FIXME: lot of cut&paste from Nix's eval.cc.
-std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen)
-{
-    str.flush();
-    checkInterrupt();
-
-    state->forceValue(v, v.determinePos(noPos));
-
-    switch (v.type()) {
-
-    case nInt:
-        str << ANSI_CYAN << v.integer << ANSI_NORMAL;
-        break;
-
-    case nBool:
-        str << ANSI_CYAN;
-        printLiteralBool(str, v.boolean);
-        str << ANSI_NORMAL;
-        break;
-
-    case nString:
-        str << ANSI_WARNING;
-        printLiteralString(str, v.string_view());
-        str << ANSI_NORMAL;
-        break;
-
-    case nPath:
-        str << ANSI_GREEN << v.path().to_string() << ANSI_NORMAL; // !!! escaping?
-        break;
-
-    case nNull:
-        str << ANSI_CYAN "null" ANSI_NORMAL;
-        break;
-
-    case nAttrs: {
-        seen.insert(&v);
-
-        bool isDrv = state->isDerivation(v);
-
-        if (isDrv) {
-            str << "«derivation ";
-            Bindings::iterator i = v.attrs->find(state->sDrvPath);
-            NixStringContext context;
-            if (i != v.attrs->end())
-                str << state->store->printStorePath(state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation"));
-            else
-                str << "???";
-            str << "»";
-        }
-
-        else if (maxDepth > 0) {
-            str << "{ ";
-
-            typedef std::map<std::string, Value *> Sorted;
-            Sorted sorted;
-            for (auto & i : *v.attrs)
-                sorted.emplace(state->symbols[i.name], i.value);
-
-            for (auto & i : sorted) {
-                printAttributeName(str, i.first);
-                str << " = ";
-                if (seen.count(i.second))
-                    str << "«repeated»";
-                else
-                    try {
-                        printValue(str, *i.second, maxDepth - 1, seen);
-                    } catch (AssertionError & e) {
-                        str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL;
-                    }
-                str << "; ";
-            }
-
-            str << "}";
-        } else
-            str << "{ ... }";
-
-        break;
-    }
-
-    case nList:
-        seen.insert(&v);
-
-        str << "[ ";
-        if (maxDepth > 0)
-            for (auto elem : v.listItems()) {
-                if (seen.count(elem))
-                    str << "«repeated»";
-                else
-                    try {
-                        printValue(str, *elem, maxDepth - 1, seen);
-                    } catch (AssertionError & e) {
-                        str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL;
-                    }
-                str << " ";
-            }
-        else
-            str << "... ";
-        str << "]";
-        break;
-
-    case nFunction:
-        if (v.isLambda()) {
-            std::ostringstream s;
-            s << state->positions[v.lambda.fun->pos];
-            str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL;
-        } else if (v.isPrimOp()) {
-            str << ANSI_MAGENTA "«primop»" ANSI_NORMAL;
-        } else if (v.isPrimOpApp()) {
-            str << ANSI_BLUE "«primop-app»" ANSI_NORMAL;
-        } else {
-            abort();
-        }
-        break;
-
-    case nFloat:
-        str << v.fpoint;
-        break;
-
-    case nThunk:
-    case nExternal:
-    default:
-        str << ANSI_RED "«unknown»" ANSI_NORMAL;
-        break;
-    }
-
-    return str;
-}
-
-
 std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create(
    const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,
    std::function<AnnotatedValues()> getValues)
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index d408f1adc..0659a2173 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -105,117 +105,23 @@ RootValue allocRootValue(Value * v)
 #endif
 }
 
-void Value::print(const SymbolTable &symbols, std::ostream &str,
-                  std::set<const void *> *seen, int depth) const
-
-{
-    checkInterrupt();
-
-    if (depth <= 0) {
-        str << "«too deep»";
-        return;
-    }
-    switch (internalType) {
-    case tInt:
-        str << integer;
-        break;
-    case tBool:
-        printLiteralBool(str, boolean);
-        break;
-    case tString:
-        printLiteralString(str, string_view());
-        break;
-    case tPath:
-        str << path().to_string(); // !!! escaping?
-        break;
-    case tNull:
-        str << "null";
-        break;
-    case tAttrs: {
-        if (seen && !attrs->empty() && !seen->insert(attrs).second)
-            str << "«repeated»";
-        else {
-            str << "{ ";
-            for (auto & i : attrs->lexicographicOrder(symbols)) {
-                str << symbols[i->name] << " = ";
-                i->value->print(symbols, str, seen, depth - 1);
-                str << "; ";
-            }
-            str << "}";
-        }
-        break;
-    }
-    case tList1:
-    case tList2:
-    case tListN:
-        if (seen && listSize() && !seen->insert(listElems()).second)
-            str << "«repeated»";
-        else {
-            str << "[ ";
-            for (auto v2 : listItems()) {
-                if (v2)
-                    v2->print(symbols, str, seen, depth - 1);
-                else
-                    str << "(nullptr)";
-                str << " ";
-            }
-            str << "]";
-        }
-        break;
-    case tThunk:
-    case tApp:
-        if (!isBlackhole()) {
-            str << "<CODE>";
-        } else {
-            // Although we know for sure that it's going to be an infinite recursion
-            // when this value is accessed _in the current context_, it's likely
-            // that the user will misinterpret a simpler «infinite recursion» output
-            // as a definitive statement about the value, while in fact it may be
-            // a valid value after `builtins.trace` and perhaps some other steps
-            // have completed.
-            str << "«potential infinite recursion»";
-        }
-        break;
-    case tLambda:
-        str << "<LAMBDA>";
-        break;
-    case tPrimOp:
-        str << "<PRIMOP>";
-        break;
-    case tPrimOpApp:
-        str << "<PRIMOP-APP>";
-        break;
-    case tExternal:
-        str << *external;
-        break;
-    case tFloat:
-        str << fpoint;
-        break;
-    default:
-        printError("Nix evaluator internal error: Value::print(): invalid value type %1%", internalType);
-        abort();
-    }
-}
-
-void Value::print(const SymbolTable &symbols, std::ostream &str,
-                  bool showRepeated, int depth) const {
-    std::set<const void *> seen;
-    print(symbols, str, showRepeated ? nullptr : &seen, depth);
-}
-
 // Pretty print types for assertion errors
 std::ostream & operator << (std::ostream & os, const ValueType t) {
     os << showType(t);
     return os;
 }
 
-std::string printValue(const EvalState & state, const Value & v)
+std::string printValue(EvalState & state, Value & v)
 {
     std::ostringstream out;
-    v.print(state.symbols, out);
+    v.print(state, out);
     return out.str();
 }
 
+void Value::print(EvalState & state, std::ostream & str, PrintOptions options)
+{
+    printValue(state, str, *this, options);
+}
 
 const Value * getPrimOp(const Value &v) {
     const Value * primOp = &v;
@@ -710,6 +616,26 @@ void PrimOp::check()
 }
 
 
+std::ostream & operator<<(std::ostream & output, PrimOp & primOp)
+{
+    output << "primop " << primOp.name;
+    return output;
+}
+
+
+PrimOp * Value::primOpAppPrimOp() const
+{
+    Value * left = primOpApp.left;
+    while (left && !left->isPrimOp()) {
+        left = left->primOpApp.left;
+    }
+
+    if (!left)
+        return nullptr;
+    return left->primOp;
+}
+
+
 void Value::mkPrimOp(PrimOp * p)
 {
     p->check();
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 5e0f1886d..9141156b1 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -84,6 +84,8 @@ struct PrimOp
     void check();
 };
 
+std::ostream & operator<<(std::ostream & output, PrimOp & primOp);
+
 /**
  * Info about a constant
  */
@@ -127,7 +129,7 @@ std::unique_ptr<ValMap> mapStaticEnvBindings(const SymbolTable & st, const Stati
 void copyContext(const Value & v, NixStringContext & context);
 
 
-std::string printValue(const EvalState & state, const Value & v);
+std::string printValue(EvalState & state, Value & v);
 std::ostream & operator << (std::ostream & os, const ValueType t);
 
 
diff --git a/src/libexpr/print-options.hh b/src/libexpr/print-options.hh
new file mode 100644
index 000000000..11ff9ae87
--- /dev/null
+++ b/src/libexpr/print-options.hh
@@ -0,0 +1,52 @@
+#pragma once
+/**
+ * @file
+ * @brief Options for printing Nix values.
+ */
+
+#include <limits>
+
+namespace nix {
+
+/**
+ * Options for printing Nix values.
+ */
+struct PrintOptions
+{
+    /**
+     * If true, output ANSI color sequences.
+     */
+    bool ansiColors = false;
+    /**
+     * If true, force values.
+     */
+    bool force = false;
+    /**
+     * If true and `force` is set, print derivations as
+     * `«derivation /nix/store/...»` instead of as attribute sets.
+     */
+    bool derivationPaths = false;
+    /**
+     * If true, track which values have been printed and skip them on
+     * subsequent encounters. Useful for self-referential values.
+     */
+    bool trackRepeated = true;
+    /**
+     * Maximum depth to evaluate to.
+     */
+    size_t maxDepth = std::numeric_limits<size_t>::max();
+    /**
+     * Maximum number of attributes in an attribute set to print.
+     */
+    size_t maxAttrs = std::numeric_limits<size_t>::max();
+    /**
+     * Maximum number of list items to print.
+     */
+    size_t maxListItems = std::numeric_limits<size_t>::max();
+    /**
+     * Maximum string length to print.
+     */
+    size_t maxStringLength = std::numeric_limits<size_t>::max();
+};
+
+}
diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc
index 53ba70bdd..db26ed4c2 100644
--- a/src/libexpr/print.cc
+++ b/src/libexpr/print.cc
@@ -1,24 +1,66 @@
-#include "print.hh"
+#include <limits>
 #include <unordered_set>
 
+#include "print.hh"
+#include "ansicolor.hh"
+#include "signals.hh"
+#include "store-api.hh"
+#include "terminal.hh"
+#include "english.hh"
+
 namespace nix {
 
-std::ostream &
-printLiteralString(std::ostream & str, const std::string_view string)
+void printElided(
+    std::ostream & output,
+    unsigned int value,
+    const std::string_view single,
+    const std::string_view plural,
+    bool ansiColors)
 {
+    if (ansiColors)
+        output << ANSI_FAINT;
+    output << " «";
+    pluralize(output, value, single, plural);
+    output << " elided»";
+    if (ansiColors)
+        output << ANSI_NORMAL;
+}
+
+
+std::ostream &
+printLiteralString(std::ostream & str, const std::string_view string, size_t maxLength, bool ansiColors)
+{
+    size_t charsPrinted = 0;
+    if (ansiColors)
+        str << ANSI_MAGENTA;
     str << "\"";
     for (auto i = string.begin(); i != string.end(); ++i) {
+        if (charsPrinted >= maxLength) {
+            str << "\"";
+            printElided(str, string.length() - charsPrinted, "byte", "bytes", ansiColors);
+            return str;
+        }
+
         if (*i == '\"' || *i == '\\') str << "\\" << *i;
         else if (*i == '\n') str << "\\n";
         else if (*i == '\r') str << "\\r";
         else if (*i == '\t') str << "\\t";
         else if (*i == '$' && *(i+1) == '{') str << "\\" << *i;
         else str << *i;
+        charsPrinted++;
     }
     str << "\"";
+    if (ansiColors)
+        str << ANSI_NORMAL;
     return str;
 }
 
+std::ostream &
+printLiteralString(std::ostream & str, const std::string_view string)
+{
+    return printLiteralString(str, string, std::numeric_limits<size_t>::max(), false);
+}
+
 std::ostream &
 printLiteralBool(std::ostream & str, bool boolean)
 {
@@ -90,5 +132,373 @@ printAttributeName(std::ostream & str, std::string_view name) {
     return str;
 }
 
+bool isImportantAttrName(const std::string& attrName)
+{
+    return attrName == "type" || attrName == "_type";
+}
+
+typedef std::pair<std::string, Value *> AttrPair;
+
+struct ImportantFirstAttrNameCmp
+{
+
+    bool operator()(const AttrPair& lhs, const AttrPair& rhs) const
+    {
+        auto lhsIsImportant = isImportantAttrName(lhs.first);
+        auto rhsIsImportant = isImportantAttrName(rhs.first);
+        return std::forward_as_tuple(!lhsIsImportant, lhs.first)
+            < std::forward_as_tuple(!rhsIsImportant, rhs.first);
+    }
+};
+
+typedef std::set<Value *> ValuesSeen;
+
+class Printer
+{
+private:
+    std::ostream & output;
+    EvalState & state;
+    PrintOptions options;
+    std::optional<ValuesSeen> seen;
+
+    void printRepeated()
+    {
+        if (options.ansiColors)
+            output << ANSI_MAGENTA;
+        output << "«repeated»";
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printNullptr()
+    {
+        if (options.ansiColors)
+            output << ANSI_MAGENTA;
+        output << "«nullptr»";
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printElided(unsigned int value, const std::string_view single, const std::string_view plural)
+    {
+        ::nix::printElided(output, value, single, plural, options.ansiColors);
+    }
+
+    void printInt(Value & v)
+    {
+        if (options.ansiColors)
+            output << ANSI_CYAN;
+        output << v.integer;
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printFloat(Value & v)
+    {
+        if (options.ansiColors)
+            output << ANSI_CYAN;
+        output << v.fpoint;
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printBool(Value & v)
+    {
+        if (options.ansiColors)
+            output << ANSI_CYAN;
+        printLiteralBool(output, v.boolean);
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printString(Value & v)
+    {
+        printLiteralString(output, v.string_view(), options.maxStringLength, options.ansiColors);
+    }
+
+    void printPath(Value & v)
+    {
+        if (options.ansiColors)
+            output << ANSI_GREEN;
+        output << v.path().to_string(); // !!! escaping?
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printNull()
+    {
+        if (options.ansiColors)
+            output << ANSI_CYAN;
+        output << "null";
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printDerivation(Value & v)
+    {
+        try {
+            Bindings::iterator i = v.attrs->find(state.sDrvPath);
+            NixStringContext context;
+            std::string storePath;
+            if (i != v.attrs->end())
+                storePath = state.store->printStorePath(state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation"));
+
+            if (options.ansiColors)
+                output << ANSI_GREEN;
+            output << "«derivation";
+            if (!storePath.empty()) {
+                output << " " << storePath;
+            }
+            output << "»";
+            if (options.ansiColors)
+                output << ANSI_NORMAL;
+        } catch (BaseError & e) {
+            printError_(e);
+        }
+    }
+
+    void printAttrs(Value & v, size_t depth)
+    {
+        if (seen && !seen->insert(&v).second) {
+            printRepeated();
+            return;
+        }
+
+        if (options.force && options.derivationPaths && state.isDerivation(v)) {
+            printDerivation(v);
+        } else if (depth < options.maxDepth) {
+            output << "{ ";
+
+            std::vector<std::pair<std::string, Value *>> sorted;
+            for (auto & i : *v.attrs)
+                sorted.emplace_back(std::pair(state.symbols[i.name], i.value));
+
+            if (options.maxAttrs == std::numeric_limits<size_t>::max())
+                std::sort(sorted.begin(), sorted.end());
+            else
+                std::sort(sorted.begin(), sorted.end(), ImportantFirstAttrNameCmp());
+
+            size_t attrsPrinted = 0;
+            for (auto & i : sorted) {
+                if (attrsPrinted >= options.maxAttrs) {
+                    printElided(sorted.size() - attrsPrinted, "attribute", "attributes");
+                    break;
+                }
+
+                printAttributeName(output, i.first);
+                output << " = ";
+                print(*i.second, depth + 1);
+                output << "; ";
+                attrsPrinted++;
+            }
+
+            output << "}";
+        } else
+            output << "{ ... }";
+    }
+
+    void printList(Value & v, size_t depth)
+    {
+        if (seen && v.listSize() && !seen->insert(&v).second) {
+            printRepeated();
+            return;
+        }
+
+        output << "[ ";
+        if (depth < options.maxDepth) {
+            size_t listItemsPrinted = 0;
+            for (auto elem : v.listItems()) {
+                if (listItemsPrinted >= options.maxListItems) {
+                    printElided(v.listSize() - listItemsPrinted, "item", "items");
+                    break;
+                }
+
+                if (elem) {
+                    print(*elem, depth + 1);
+                } else {
+                    printNullptr();
+                }
+                output << " ";
+                listItemsPrinted++;
+            }
+        }
+        else
+            output << "... ";
+        output << "]";
+    }
+
+    void printFunction(Value & v)
+    {
+        if (options.ansiColors)
+            output << ANSI_BLUE;
+        output << "«";
+
+        if (v.isLambda()) {
+            output << "lambda";
+            if (v.lambda.fun) {
+                if (v.lambda.fun->name) {
+                    output << " " << state.symbols[v.lambda.fun->name];
+                }
+
+                std::ostringstream s;
+                s << state.positions[v.lambda.fun->pos];
+                output << " @ " << filterANSIEscapes(s.str());
+            }
+        } else if (v.isPrimOp()) {
+            if (v.primOp)
+                output << *v.primOp;
+            else
+                output << "primop";
+        } else if (v.isPrimOpApp()) {
+            output << "partially applied ";
+            auto primOp = v.primOpAppPrimOp();
+            if (primOp)
+                output << *primOp;
+            else
+                output << "primop";
+        } else {
+            abort();
+        }
+
+        output << "»";
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printThunk(Value & v)
+    {
+        if (v.isBlackhole()) {
+            // Although we know for sure that it's going to be an infinite recursion
+            // when this value is accessed _in the current context_, it's likely
+            // that the user will misinterpret a simpler «infinite recursion» output
+            // as a definitive statement about the value, while in fact it may be
+            // a valid value after `builtins.trace` and perhaps some other steps
+            // have completed.
+            if (options.ansiColors)
+                output << ANSI_RED;
+            output << "«potential infinite recursion»";
+            if (options.ansiColors)
+                output << ANSI_NORMAL;
+        } else if (v.isThunk() || v.isApp()) {
+            if (options.ansiColors)
+                    output << ANSI_MAGENTA;
+            output << "«thunk»";
+            if (options.ansiColors)
+                    output << ANSI_NORMAL;
+        } else {
+            abort();
+        }
+    }
+
+    void printExternal(Value & v)
+    {
+        v.external->print(output);
+    }
+
+    void printUnknown()
+    {
+        if (options.ansiColors)
+            output << ANSI_RED;
+        output << "«unknown»";
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void printError_(BaseError & e)
+    {
+        if (options.ansiColors)
+            output << ANSI_RED;
+        output << "«" << e.msg() << "»";
+        if (options.ansiColors)
+            output << ANSI_NORMAL;
+    }
+
+    void print(Value & v, size_t depth)
+    {
+        output.flush();
+        checkInterrupt();
+
+        if (options.force) {
+            try {
+                state.forceValue(v, v.determinePos(noPos));
+            } catch (BaseError & e) {
+                printError_(e);
+                return;
+            }
+        }
+
+        switch (v.type()) {
+
+        case nInt:
+            printInt(v);
+            break;
+
+        case nFloat:
+            printFloat(v);
+            break;
+
+        case nBool:
+            printBool(v);
+            break;
+
+        case nString:
+            printString(v);
+            break;
+
+        case nPath:
+            printPath(v);
+            break;
+
+        case nNull:
+            printNull();
+            break;
+
+        case nAttrs:
+            printAttrs(v, depth);
+            break;
+
+        case nList:
+            printList(v, depth);
+            break;
+
+        case nFunction:
+            printFunction(v);
+            break;
+
+        case nThunk:
+            printThunk(v);
+            break;
+
+        case nExternal:
+            printExternal(v);
+            break;
+
+        default:
+            printUnknown();
+            break;
+        }
+    }
+
+public:
+    Printer(std::ostream & output, EvalState & state, PrintOptions options)
+        : output(output), state(state), options(options) { }
+
+    void print(Value & v)
+    {
+        if (options.trackRepeated) {
+            seen.emplace();
+        } else {
+            seen.reset();
+        }
+
+        ValuesSeen seen;
+        print(v, 0);
+    }
+};
+
+void printValue(EvalState & state, std::ostream & output, Value & v, PrintOptions options)
+{
+    Printer(output, state, options).print(v);
+}
 
 }
diff --git a/src/libexpr/print.hh b/src/libexpr/print.hh
index abf830864..40207d777 100644
--- a/src/libexpr/print.hh
+++ b/src/libexpr/print.hh
@@ -9,6 +9,9 @@
 
 #include <iostream>
 
+#include "eval.hh"
+#include "print-options.hh"
+
 namespace nix {
 
 /**
@@ -16,6 +19,7 @@ namespace nix {
  *
  * Quotes and fairly minimal escaping are added.
  *
+ * @param o The output stream to print to
  * @param s The logical string
  */
 std::ostream & printLiteralString(std::ostream & o, std::string_view s);
@@ -53,4 +57,6 @@ bool isReservedKeyword(const std::string_view str);
  */
 std::ostream & printIdentifier(std::ostream & o, std::string_view s);
 
+void printValue(EvalState & state, std::ostream & str, Value & v, PrintOptions options = PrintOptions {});
+
 }
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index c65b336b0..214d52271 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -9,6 +9,7 @@
 #include "value/context.hh"
 #include "input-accessor.hh"
 #include "source-path.hh"
+#include "print-options.hh"
 
 #if HAVE_BOEHMGC
 #include <gc/gc_allocator.h>
@@ -70,7 +71,7 @@ struct Pos;
 class StorePath;
 class EvalState;
 class XMLWriter;
-
+class Printer;
 
 typedef int64_t NixInt;
 typedef double NixFloat;
@@ -82,6 +83,7 @@ typedef double NixFloat;
 class ExternalValueBase
 {
     friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
+    friend class Printer;
     protected:
     /**
      * Print out the value
@@ -139,11 +141,9 @@ private:
 
     friend std::string showType(const Value & v);
 
-    void print(const SymbolTable &symbols, std::ostream &str, std::set<const void *> *seen, int depth) const;
-
 public:
 
-    void print(const SymbolTable &symbols, std::ostream &str, bool showRepeated = false, int depth = INT_MAX) const;
+    void print(EvalState &state, std::ostream &str, PrintOptions options = PrintOptions {});
 
     // Functions needed to distinguish the type
     // These should be removed eventually, by putting the functionality that's
@@ -364,10 +364,15 @@ public:
     inline void mkPrimOpApp(Value * l, Value * r)
     {
         internalType = tPrimOpApp;
-        app.left = l;
-        app.right = r;
+        primOpApp.left = l;
+        primOpApp.right = r;
     }
 
+    /**
+     * For a `tPrimOpApp` value, get the original `PrimOp` value.
+     */
+    PrimOp * primOpAppPrimOp() const;
+
     inline void mkExternal(ExternalValueBase * e)
     {
         clearValue();
diff --git a/src/libutil/english.cc b/src/libutil/english.cc
new file mode 100644
index 000000000..8c93c9156
--- /dev/null
+++ b/src/libutil/english.cc
@@ -0,0 +1,18 @@
+#include "english.hh"
+
+namespace nix {
+
+std::ostream & pluralize(
+    std::ostream & output,
+    unsigned int count,
+    const std::string_view single,
+    const std::string_view plural)
+{
+    if (count == 1)
+        output << "1 " << single;
+    else
+        output << count << " " << plural;
+    return output;
+}
+
+}
diff --git a/src/libutil/english.hh b/src/libutil/english.hh
new file mode 100644
index 000000000..9c6c93571
--- /dev/null
+++ b/src/libutil/english.hh
@@ -0,0 +1,18 @@
+#pragma once
+
+#include <iostream>
+
+namespace nix {
+
+/**
+ * Pluralize a given value.
+ *
+ * If `count == 1`, prints `1 {single}` to `output`, otherwise prints `{count} {plural}`.
+ */
+std::ostream & pluralize(
+    std::ostream & output,
+    unsigned int count,
+    const std::string_view single,
+    const std::string_view plural);
+
+}
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index 9f4d063d2..3d07cab7a 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -8,6 +8,8 @@
 #include "eval.hh"
 #include "eval-inline.hh"
 #include "profiles.hh"
+#include "print-ambiguous.hh"
+#include <limits>
 
 
 namespace nix {
@@ -106,7 +108,8 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
        environment. */
     auto manifestFile = ({
         std::ostringstream str;
-        manifest.print(state.symbols, str, true);
+        std::set<const void *> seen;
+        printAmbiguous(manifest, state.symbols, str, &seen, std::numeric_limits<int>::max());
         // TODO with C++20 we can use str.view() instead and avoid copy.
         std::string str2 = str.str();
         StringSource source { str2 };
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index ab590b3a6..9b36dccc6 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -56,7 +56,7 @@ void processExpr(EvalState & state, const Strings & attrPaths,
                 std::cout << std::endl;
             } else {
                 if (strict) state.forceValueDeep(vRes);
-                vRes.print(state.symbols, std::cout);
+                vRes.print(state, std::cout);
                 std::cout << std::endl;
             }
         } else {
diff --git a/tests/functional/lang/eval-okay-print.err.exp b/tests/functional/lang/eval-okay-print.err.exp
index 3fc99be3e..80aa17c6e 100644
--- a/tests/functional/lang/eval-okay-print.err.exp
+++ b/tests/functional/lang/eval-okay-print.err.exp
@@ -1 +1 @@
-trace: [ <CODE> ]
+trace: [ «thunk» ]
diff --git a/tests/functional/lang/eval-okay-print.exp b/tests/functional/lang/eval-okay-print.exp
index 0d960fb70..aa1b2379e 100644
--- a/tests/functional/lang/eval-okay-print.exp
+++ b/tests/functional/lang/eval-okay-print.exp
@@ -1 +1 @@
-[ null <PRIMOP> <PRIMOP-APP> <LAMBDA> [ [ «repeated» ] ] ]
+[ null «primop toString» «partially applied primop deepSeq» «lambda @ /pwd/lang/eval-okay-print.nix:1:61» [ [ «repeated» ] ] ]
diff --git a/tests/functional/lang/eval-okay-repeated-empty-attrs.exp b/tests/functional/lang/eval-okay-repeated-empty-attrs.exp
new file mode 100644
index 000000000..d21e6db6b
--- /dev/null
+++ b/tests/functional/lang/eval-okay-repeated-empty-attrs.exp
@@ -0,0 +1 @@
+[ { } { } ]
diff --git a/tests/functional/lang/eval-okay-repeated-empty-attrs.nix b/tests/functional/lang/eval-okay-repeated-empty-attrs.nix
new file mode 100644
index 000000000..030a3b85c
--- /dev/null
+++ b/tests/functional/lang/eval-okay-repeated-empty-attrs.nix
@@ -0,0 +1,2 @@
+# Tests that empty attribute sets are not printed as `«repeated»`.
+[ {} {} ]
diff --git a/tests/functional/lang/eval-okay-repeated-empty-list.exp b/tests/functional/lang/eval-okay-repeated-empty-list.exp
new file mode 100644
index 000000000..701fc7e20
--- /dev/null
+++ b/tests/functional/lang/eval-okay-repeated-empty-list.exp
@@ -0,0 +1 @@
+[ [ ] [ ] ]
diff --git a/tests/functional/lang/eval-okay-repeated-empty-list.nix b/tests/functional/lang/eval-okay-repeated-empty-list.nix
new file mode 100644
index 000000000..376c51be8
--- /dev/null
+++ b/tests/functional/lang/eval-okay-repeated-empty-list.nix
@@ -0,0 +1 @@
+[ [] [] ]
diff --git a/tests/unit/libexpr/value/print.cc b/tests/unit/libexpr/value/print.cc
index a4f6fc014..98131112e 100644
--- a/tests/unit/libexpr/value/print.cc
+++ b/tests/unit/libexpr/value/print.cc
@@ -1,6 +1,7 @@
 #include "tests/libexpr.hh"
 
 #include "value.hh"
+#include "print.hh"
 
 namespace nix {
 
@@ -12,7 +13,7 @@ struct ValuePrintingTests : LibExprTest
     void test(Value v, std::string_view expected, A... args)
     {
         std::stringstream out;
-        v.print(state.symbols, out, args...);
+        v.print(state, out, args...);
         ASSERT_EQ(out.str(), expected);
     }
 };
@@ -84,7 +85,7 @@ TEST_F(ValuePrintingTests, tList)
     vList.bigList.elems[1] = &vTwo;
     vList.bigList.size = 3;
 
-    test(vList, "[ 1 2 (nullptr) ]");
+    test(vList, "[ 1 2 «nullptr» ]");
 }
 
 TEST_F(ValuePrintingTests, vThunk)
@@ -92,7 +93,7 @@ TEST_F(ValuePrintingTests, vThunk)
     Value vThunk;
     vThunk.mkThunk(nullptr, nullptr);
 
-    test(vThunk, "<CODE>");
+    test(vThunk, "«thunk»");
 }
 
 TEST_F(ValuePrintingTests, vApp)
@@ -100,32 +101,55 @@ TEST_F(ValuePrintingTests, vApp)
     Value vApp;
     vApp.mkApp(nullptr, nullptr);
 
-    test(vApp, "<CODE>");
+    test(vApp, "«thunk»");
 }
 
 TEST_F(ValuePrintingTests, vLambda)
 {
-    Value vLambda;
-    vLambda.mkLambda(nullptr, nullptr);
+    Env env {
+        .up = nullptr,
+        .values = { }
+    };
+    PosTable::Origin origin((std::monostate()));
+    auto posIdx = state.positions.add(origin, 1, 1);
+    auto body = ExprInt(0);
+    auto formals = Formals {};
 
-    test(vLambda, "<LAMBDA>");
+    ExprLambda eLambda(posIdx, createSymbol("a"), &formals, &body);
+
+    Value vLambda;
+    vLambda.mkLambda(&env, &eLambda);
+
+    test(vLambda, "«lambda @ «none»:1:1»");
+
+    eLambda.setName(createSymbol("puppy"));
+
+    test(vLambda, "«lambda puppy @ «none»:1:1»");
 }
 
 TEST_F(ValuePrintingTests, vPrimOp)
 {
     Value vPrimOp;
-    PrimOp primOp{};
+    PrimOp primOp{
+        .name = "puppy"
+    };
     vPrimOp.mkPrimOp(&primOp);
 
-    test(vPrimOp, "<PRIMOP>");
+    test(vPrimOp, "«primop puppy»");
 }
 
 TEST_F(ValuePrintingTests, vPrimOpApp)
 {
-    Value vPrimOpApp;
-    vPrimOpApp.mkPrimOpApp(nullptr, nullptr);
+    PrimOp primOp{
+        .name = "puppy"
+    };
+    Value vPrimOp;
+    vPrimOp.mkPrimOp(&primOp);
 
-    test(vPrimOpApp, "<PRIMOP-APP>");
+    Value vPrimOpApp;
+    vPrimOpApp.mkPrimOpApp(&vPrimOp, nullptr);
+
+    test(vPrimOpApp, "«partially applied primop puppy»");
 }
 
 TEST_F(ValuePrintingTests, vExternal)
@@ -176,9 +200,14 @@ TEST_F(ValuePrintingTests, depthAttrs)
     Value vTwo;
     vTwo.mkInt(2);
 
+    BindingsBuilder builderEmpty(state, state.allocBindings(0));
+    Value vAttrsEmpty;
+    vAttrsEmpty.mkAttrs(builderEmpty.finish());
+
     BindingsBuilder builder(state, state.allocBindings(10));
     builder.insert(state.symbols.create("one"), &vOne);
     builder.insert(state.symbols.create("two"), &vTwo);
+    builder.insert(state.symbols.create("nested"), &vAttrsEmpty);
 
     Value vAttrs;
     vAttrs.mkAttrs(builder.finish());
@@ -191,10 +220,10 @@ TEST_F(ValuePrintingTests, depthAttrs)
     Value vNested;
     vNested.mkAttrs(builder2.finish());
 
-    test(vNested, "{ nested = «too deep»; one = «too deep»; two = «too deep»; }", false, 1);
-    test(vNested, "{ nested = { one = «too deep»; two = «too deep»; }; one = 1; two = 2; }", false, 2);
-    test(vNested, "{ nested = { one = 1; two = 2; }; one = 1; two = 2; }", false, 3);
-    test(vNested, "{ nested = { one = 1; two = 2; }; one = 1; two = 2; }", false, 4);
+    test(vNested, "{ nested = { ... }; one = 1; two = 2; }", PrintOptions { .maxDepth = 1 });
+    test(vNested, "{ nested = { nested = { ... }; one = 1; two = 2; }; one = 1; two = 2; }", PrintOptions { .maxDepth = 2 });
+    test(vNested, "{ nested = { nested = { }; one = 1; two = 2; }; one = 1; two = 2; }", PrintOptions { .maxDepth = 3 });
+    test(vNested, "{ nested = { nested = { }; one = 1; two = 2; }; one = 1; two = 2; }", PrintOptions { .maxDepth = 4 });
 }
 
 TEST_F(ValuePrintingTests, depthList)
@@ -227,11 +256,561 @@ TEST_F(ValuePrintingTests, depthList)
     vList.bigList.elems[2] = &vNested;
     vList.bigList.size = 3;
 
-    test(vList, "[ «too deep» «too deep» «too deep» ]", false, 1);
-    test(vList, "[ 1 2 { nested = «too deep»; one = «too deep»; two = «too deep»; } ]", false, 2);
-    test(vList, "[ 1 2 { nested = { one = «too deep»; two = «too deep»; }; one = 1; two = 2; } ]", false, 3);
-    test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", false, 4);
-    test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", false, 5);
+    test(vList, "[ 1 2 { ... } ]", PrintOptions { .maxDepth = 1 });
+    test(vList, "[ 1 2 { nested = { ... }; one = 1; two = 2; } ]", PrintOptions { .maxDepth = 2 });
+    test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", PrintOptions { .maxDepth = 3 });
+    test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", PrintOptions { .maxDepth = 4 });
+    test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", PrintOptions { .maxDepth = 5 });
+}
+
+struct StringPrintingTests : LibExprTest
+{
+    template<class... A>
+    void test(std::string_view literal, std::string_view expected, unsigned int maxLength, A... args)
+    {
+        Value v;
+        v.mkString(literal);
+
+        std::stringstream out;
+        printValue(state, out, v, PrintOptions {
+            .maxStringLength = maxLength
+        });
+        ASSERT_EQ(out.str(), expected);
+    }
+};
+
+TEST_F(StringPrintingTests, maxLengthTruncation)
+{
+    test("abcdefghi", "\"abcdefghi\"", 10);
+    test("abcdefghij", "\"abcdefghij\"", 10);
+    test("abcdefghijk", "\"abcdefghij\" «1 byte elided»", 10);
+    test("abcdefghijkl", "\"abcdefghij\" «2 bytes elided»", 10);
+    test("abcdefghijklm", "\"abcdefghij\" «3 bytes elided»", 10);
+}
+
+// Check that printing an attrset shows 'important' attributes like `type`
+// first, but only reorder the attrs when we have a maxAttrs budget.
+TEST_F(ValuePrintingTests, attrsTypeFirst)
+{
+    Value vType;
+    vType.mkString("puppy");
+
+    Value vApple;
+    vApple.mkString("apple");
+
+    BindingsBuilder builder(state, state.allocBindings(10));
+    builder.insert(state.symbols.create("type"), &vType);
+    builder.insert(state.symbols.create("apple"), &vApple);
+
+    Value vAttrs;
+    vAttrs.mkAttrs(builder.finish());
+
+    test(vAttrs,
+         "{ type = \"puppy\"; apple = \"apple\"; }",
+         PrintOptions {
+            .maxAttrs = 100
+         });
+
+    test(vAttrs,
+         "{ apple = \"apple\"; type = \"puppy\"; }",
+         PrintOptions { });
+}
+
+TEST_F(ValuePrintingTests, ansiColorsInt)
+{
+    Value v;
+    v.mkInt(10);
+
+    test(v,
+         ANSI_CYAN "10" ANSI_NORMAL,
+         PrintOptions {
+             .ansiColors = true
+         });
+}
+
+TEST_F(ValuePrintingTests, ansiColorsFloat)
+{
+    Value v;
+    v.mkFloat(1.6);
+
+    test(v,
+         ANSI_CYAN "1.6" ANSI_NORMAL,
+         PrintOptions {
+             .ansiColors = true
+         });
+}
+
+TEST_F(ValuePrintingTests, ansiColorsBool)
+{
+    Value v;
+    v.mkBool(true);
+
+    test(v,
+         ANSI_CYAN "true" ANSI_NORMAL,
+         PrintOptions {
+             .ansiColors = true
+         });
+}
+
+TEST_F(ValuePrintingTests, ansiColorsString)
+{
+    Value v;
+    v.mkString("puppy");
+
+    test(v,
+         ANSI_MAGENTA "\"puppy\"" ANSI_NORMAL,
+         PrintOptions {
+             .ansiColors = true
+        });
+}
+
+TEST_F(ValuePrintingTests, ansiColorsStringElided)
+{
+    Value v;
+    v.mkString("puppy");
+
+    test(v,
+         ANSI_MAGENTA "\"pup\"" ANSI_FAINT " «2 bytes elided»" ANSI_NORMAL,
+         PrintOptions {
+             .ansiColors = true,
+             .maxStringLength = 3
+         });
+}
+
+TEST_F(ValuePrintingTests, ansiColorsPath)
+{
+    Value v;
+    v.mkPath(state.rootPath(CanonPath("puppy")));
+
+    test(v,
+         ANSI_GREEN "/puppy" ANSI_NORMAL,
+         PrintOptions {
+             .ansiColors = true
+         });
+}
+
+TEST_F(ValuePrintingTests, ansiColorsNull)
+{
+    Value v;
+    v.mkNull();
+
+    test(v,
+         ANSI_CYAN "null" ANSI_NORMAL,
+         PrintOptions {
+             .ansiColors = true
+         });
+}
+
+TEST_F(ValuePrintingTests, ansiColorsAttrs)
+{
+    Value vOne;
+    vOne.mkInt(1);
+
+    Value vTwo;
+    vTwo.mkInt(2);
+
+    BindingsBuilder builder(state, state.allocBindings(10));
+    builder.insert(state.symbols.create("one"), &vOne);
+    builder.insert(state.symbols.create("two"), &vTwo);
+
+    Value vAttrs;
+    vAttrs.mkAttrs(builder.finish());
+
+    test(vAttrs,
+         "{ one = " ANSI_CYAN "1" ANSI_NORMAL "; two = " ANSI_CYAN "2" ANSI_NORMAL "; }",
+         PrintOptions {
+             .ansiColors = true
+         });
+}
+
+TEST_F(ValuePrintingTests, ansiColorsDerivation)
+{
+    Value vDerivation;
+    vDerivation.mkString("derivation");
+
+    BindingsBuilder builder(state, state.allocBindings(10));
+    builder.insert(state.sType, &vDerivation);
+
+    Value vAttrs;
+    vAttrs.mkAttrs(builder.finish());
+
+    test(vAttrs,
+         ANSI_GREEN "«derivation»" ANSI_NORMAL,
+         PrintOptions {
+             .ansiColors = true,
+             .force = true,
+             .derivationPaths = true
+         });
+
+    test(vAttrs,
+         "{ type = " ANSI_MAGENTA "\"derivation\"" ANSI_NORMAL "; }",
+         PrintOptions {
+             .ansiColors = true,
+             .force = true
+         });
+}
+
+TEST_F(ValuePrintingTests, ansiColorsError)
+{
+    Value throw_ = state.getBuiltin("throw");
+    Value message;
+    message.mkString("uh oh!");
+    Value vError;
+    vError.mkApp(&throw_, &message);
+
+    test(vError,
+         ANSI_RED
+         "«"
+         ANSI_RED
+         "error:"
+         ANSI_NORMAL
+         "\n       … while calling the '"
+         ANSI_MAGENTA
+         "throw"
+         ANSI_NORMAL
+         "' builtin\n\n       "
+         ANSI_RED
+         "error:"
+         ANSI_NORMAL
+         " uh oh!»"
+         ANSI_NORMAL,
+         PrintOptions {
+             .ansiColors = true,
+             .force = true,
+         });
+}
+
+TEST_F(ValuePrintingTests, ansiColorsDerivationError)
+{
+    Value throw_ = state.getBuiltin("throw");
+    Value message;
+    message.mkString("uh oh!");
+    Value vError;
+    vError.mkApp(&throw_, &message);
+
+    Value vDerivation;
+    vDerivation.mkString("derivation");
+
+    BindingsBuilder builder(state, state.allocBindings(10));
+    builder.insert(state.sType, &vDerivation);
+    builder.insert(state.sDrvPath, &vError);
+
+    Value vAttrs;
+    vAttrs.mkAttrs(builder.finish());
+
+    test(vAttrs,
+         "{ drvPath = "
+         ANSI_RED
+         "«"
+         ANSI_RED
+         "error:"
+         ANSI_NORMAL
+         "\n       … while calling the '"
+         ANSI_MAGENTA
+         "throw"
+         ANSI_NORMAL
+         "' builtin\n\n       "
+         ANSI_RED
+         "error:"
+         ANSI_NORMAL
+         " uh oh!»"
+         ANSI_NORMAL
+         "; type = "
+         ANSI_MAGENTA
+         "\"derivation\""
+         ANSI_NORMAL
+         "; }",
+         PrintOptions {
+             .ansiColors = true,
+             .force = true
+         });
+
+    test(vAttrs,
+         ANSI_RED
+         "«"
+         ANSI_RED
+         "error:"
+         ANSI_NORMAL
+         "\n       … while calling the '"
+         ANSI_MAGENTA
+         "throw"
+         ANSI_NORMAL
+         "' builtin\n\n       "
+         ANSI_RED
+         "error:"
+         ANSI_NORMAL
+         " uh oh!»"
+         ANSI_NORMAL,
+         PrintOptions {
+             .ansiColors = true,
+             .force = true,
+             .derivationPaths = true,
+         });
+}
+
+TEST_F(ValuePrintingTests, ansiColorsAssert)
+{
+    ExprVar eFalse(state.symbols.create("false"));
+    eFalse.bindVars(state, state.staticBaseEnv);
+    ExprInt eInt(1);
+
+    ExprAssert expr(noPos, &eFalse, &eInt);
+
+    Value v;
+    state.mkThunk_(v, &expr);
+
+    test(v,
+         ANSI_RED "«" ANSI_RED "error:" ANSI_NORMAL " assertion '" ANSI_MAGENTA "false" ANSI_NORMAL "' failed»" ANSI_NORMAL,
+         PrintOptions {
+             .ansiColors = true,
+             .force = true
+         });
+}
+
+TEST_F(ValuePrintingTests, ansiColorsList)
+{
+    Value vOne;
+    vOne.mkInt(1);
+
+    Value vTwo;
+    vTwo.mkInt(2);
+
+    Value vList;
+    state.mkList(vList, 5);
+    vList.bigList.elems[0] = &vOne;
+    vList.bigList.elems[1] = &vTwo;
+    vList.bigList.size = 3;
+
+    test(vList,
+         "[ " ANSI_CYAN "1" ANSI_NORMAL " " ANSI_CYAN "2" ANSI_NORMAL " " ANSI_MAGENTA "«nullptr»" ANSI_NORMAL " ]",
+         PrintOptions {
+             .ansiColors = true
+         });
+}
+
+TEST_F(ValuePrintingTests, ansiColorsLambda)
+{
+    Env env {
+        .up = nullptr,
+        .values = { }
+    };
+    PosTable::Origin origin((std::monostate()));
+    auto posIdx = state.positions.add(origin, 1, 1);
+    auto body = ExprInt(0);
+    auto formals = Formals {};
+
+    ExprLambda eLambda(posIdx, createSymbol("a"), &formals, &body);
+
+    Value vLambda;
+    vLambda.mkLambda(&env, &eLambda);
+
+    test(vLambda,
+         ANSI_BLUE "«lambda @ «none»:1:1»" ANSI_NORMAL,
+         PrintOptions {
+             .ansiColors = true,
+             .force = true
+         });
+
+    eLambda.setName(createSymbol("puppy"));
+
+    test(vLambda,
+         ANSI_BLUE "«lambda puppy @ «none»:1:1»" ANSI_NORMAL,
+         PrintOptions {
+             .ansiColors = true,
+             .force = true
+         });
+}
+
+TEST_F(ValuePrintingTests, ansiColorsPrimOp)
+{
+    PrimOp primOp{
+        .name = "puppy"
+    };
+    Value v;
+    v.mkPrimOp(&primOp);
+
+    test(v,
+         ANSI_BLUE "«primop puppy»" ANSI_NORMAL,
+         PrintOptions {
+             .ansiColors = true
+         });
+}
+
+TEST_F(ValuePrintingTests, ansiColorsPrimOpApp)
+{
+    PrimOp primOp{
+        .name = "puppy"
+    };
+    Value vPrimOp;
+    vPrimOp.mkPrimOp(&primOp);
+
+    Value v;
+    v.mkPrimOpApp(&vPrimOp, nullptr);
+
+    test(v,
+         ANSI_BLUE "«partially applied primop puppy»" ANSI_NORMAL,
+         PrintOptions {
+             .ansiColors = true
+         });
+}
+
+TEST_F(ValuePrintingTests, ansiColorsThunk)
+{
+    Value v;
+    v.mkThunk(nullptr, nullptr);
+
+    test(v,
+         ANSI_MAGENTA "«thunk»" ANSI_NORMAL,
+         PrintOptions {
+             .ansiColors = true
+         });
+}
+
+TEST_F(ValuePrintingTests, ansiColorsBlackhole)
+{
+    Value v;
+    v.mkBlackhole();
+
+    test(v,
+         ANSI_RED "«potential infinite recursion»" ANSI_NORMAL,
+         PrintOptions {
+             .ansiColors = true
+         });
+}
+
+TEST_F(ValuePrintingTests, ansiColorsAttrsRepeated)
+{
+    BindingsBuilder emptyBuilder(state, state.allocBindings(1));
+
+    Value vEmpty;
+    vEmpty.mkAttrs(emptyBuilder.finish());
+
+    BindingsBuilder builder(state, state.allocBindings(10));
+    builder.insert(state.symbols.create("a"), &vEmpty);
+    builder.insert(state.symbols.create("b"), &vEmpty);
+
+    Value vAttrs;
+    vAttrs.mkAttrs(builder.finish());
+
+    test(vAttrs,
+         "{ a = { }; b = " ANSI_MAGENTA "«repeated»" ANSI_NORMAL "; }",
+         PrintOptions {
+             .ansiColors = true
+         });
+}
+
+TEST_F(ValuePrintingTests, ansiColorsListRepeated)
+{
+    BindingsBuilder emptyBuilder(state, state.allocBindings(1));
+
+    Value vEmpty;
+    vEmpty.mkAttrs(emptyBuilder.finish());
+
+    Value vList;
+    state.mkList(vList, 3);
+    vList.bigList.elems[0] = &vEmpty;
+    vList.bigList.elems[1] = &vEmpty;
+    vList.bigList.size = 2;
+
+    test(vList,
+         "[ { } " ANSI_MAGENTA "«repeated»" ANSI_NORMAL " ]",
+         PrintOptions {
+             .ansiColors = true
+         });
+}
+
+TEST_F(ValuePrintingTests, listRepeated)
+{
+    BindingsBuilder emptyBuilder(state, state.allocBindings(1));
+
+    Value vEmpty;
+    vEmpty.mkAttrs(emptyBuilder.finish());
+
+    Value vList;
+    state.mkList(vList, 3);
+    vList.bigList.elems[0] = &vEmpty;
+    vList.bigList.elems[1] = &vEmpty;
+    vList.bigList.size = 2;
+
+    test(vList, "[ { } «repeated» ]", PrintOptions { });
+    test(vList,
+         "[ { } { } ]",
+         PrintOptions {
+             .trackRepeated = false
+         });
+}
+
+TEST_F(ValuePrintingTests, ansiColorsAttrsElided)
+{
+    Value vOne;
+    vOne.mkInt(1);
+
+    Value vTwo;
+    vTwo.mkInt(2);
+
+    BindingsBuilder builder(state, state.allocBindings(10));
+    builder.insert(state.symbols.create("one"), &vOne);
+    builder.insert(state.symbols.create("two"), &vTwo);
+
+    Value vAttrs;
+    vAttrs.mkAttrs(builder.finish());
+
+    test(vAttrs,
+         "{ one = " ANSI_CYAN "1" ANSI_NORMAL "; " ANSI_FAINT " «1 attribute elided»" ANSI_NORMAL "}",
+         PrintOptions {
+             .ansiColors = true,
+             .maxAttrs = 1
+         });
+
+    Value vThree;
+    vThree.mkInt(3);
+
+    builder.insert(state.symbols.create("three"), &vThree);
+    vAttrs.mkAttrs(builder.finish());
+
+    test(vAttrs,
+         "{ one = " ANSI_CYAN "1" ANSI_NORMAL "; " ANSI_FAINT " «2 attributes elided»" ANSI_NORMAL "}",
+         PrintOptions {
+             .ansiColors = true,
+             .maxAttrs = 1
+         });
+}
+
+TEST_F(ValuePrintingTests, ansiColorsListElided)
+{
+    BindingsBuilder emptyBuilder(state, state.allocBindings(1));
+
+    Value vOne;
+    vOne.mkInt(1);
+
+    Value vTwo;
+    vTwo.mkInt(2);
+
+    Value vList;
+    state.mkList(vList, 4);
+    vList.bigList.elems[0] = &vOne;
+    vList.bigList.elems[1] = &vTwo;
+    vList.bigList.size = 2;
+
+    test(vList,
+         "[ " ANSI_CYAN "1" ANSI_NORMAL " " ANSI_FAINT " «1 item elided»" ANSI_NORMAL "]",
+         PrintOptions {
+             .ansiColors = true,
+             .maxListItems = 1
+         });
+
+    Value vThree;
+    vThree.mkInt(3);
+
+    vList.bigList.elems[2] = &vThree;
+    vList.bigList.size = 3;
+
+    test(vList,
+         "[ " ANSI_CYAN "1" ANSI_NORMAL " " ANSI_FAINT " «2 items elided»" ANSI_NORMAL "]",
+         PrintOptions {
+             .ansiColors = true,
+             .maxListItems = 1
+         });
 }
 
 } // namespace nix

From df84dd4d8dd3fd6381ac2ca3064432ab31a16b79 Mon Sep 17 00:00:00 2001
From: Rebecca Turner <rbt@sent.as>
Date: Tue, 9 Jan 2024 11:13:45 -0800
Subject: [PATCH 333/421] Restore ambiguous value printer for `nix-instantiate`

The Nix team has requested that this output format remain unchanged.
I've added a warning to the man page explaining that `nix-instantiate
--eval` output will not parse correctly in many situations.
---
 doc/manual/src/command-ref/nix-instantiate.md |  80 ++++++++++----
 src/libexpr/print-ambiguous.cc                | 100 ++++++++++++++++++
 src/libexpr/print-ambiguous.hh                |  24 +++++
 src/nix-env/user-env.cc                       |   3 +-
 src/nix-instantiate/nix-instantiate.cc        |   6 +-
 tests/functional/lang/eval-okay-print.exp     |   2 +-
 6 files changed, 189 insertions(+), 26 deletions(-)
 create mode 100644 src/libexpr/print-ambiguous.cc
 create mode 100644 src/libexpr/print-ambiguous.hh

diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md
index e1b4a3e80..483150aa8 100644
--- a/doc/manual/src/command-ref/nix-instantiate.md
+++ b/doc/manual/src/command-ref/nix-instantiate.md
@@ -35,13 +35,50 @@ standard input.
 
   - `--parse`\
     Just parse the input files, and print their abstract syntax trees on
-    standard output in ATerm format.
+    standard output as a Nix expression.
 
   - `--eval`\
     Just parse and evaluate the input files, and print the resulting
     values on standard output. No instantiation of store derivations
     takes place.
 
+    > **Warning**
+    >
+    > This option produces ambiguous output which is not suitable for machine
+    > consumption. For example, these two Nix expressions print the same result
+    > despite having different types:
+    >
+    > ```console
+    > $ nix-instantiate --eval --expr '{ a = {}; }'
+    > { a = <CODE>; }
+    > $ nix-instantiate --eval --expr '{ a = <CODE>; }'
+    > { a = <CODE>; }
+    > ```
+    >
+    > For human-readable output, `nix eval` (experimental) is more informative:
+    >
+    > ```console
+    > $ nix-instantiate --eval --expr 'a: a'
+    > <LAMBDA>
+    > $ nix eval --expr 'a: a'
+    > «lambda @ «string»:1:1»
+    > ```
+    >
+    > For machine-readable output, the `--xml` option produces unambiguous
+    > output:
+    >
+    > ```console
+    > $ nix-instantiate --eval --xml --expr '{ foo = <CODE>; }'
+    > <?xml version='1.0' encoding='utf-8'?>
+    > <expr>
+    >   <attrs>
+    >     <attr column="3" line="1" name="foo">
+    >       <unevaluated />
+    >     </attr>
+    >   </attrs>
+    > </expr>
+    > ```
+
   - `--find-file`\
     Look up the given files in Nix’s search path (as specified by the
     `NIX_PATH` environment variable). If found, print the corresponding
@@ -61,11 +98,11 @@ standard input.
 
   - `--json`\
     When used with `--eval`, print the resulting value as an JSON
-    representation of the abstract syntax tree rather than as an ATerm.
+    representation of the abstract syntax tree rather than as a Nix expression.
 
   - `--xml`\
     When used with `--eval`, print the resulting value as an XML
-    representation of the abstract syntax tree rather than as an ATerm.
+    representation of the abstract syntax tree rather than as a Nix expression.
     The schema is the same as that used by the [`toXML`
     built-in](../language/builtins.md).
 
@@ -133,28 +170,29 @@ $ nix-instantiate --eval --xml --expr '1 + 2'
 The difference between non-strict and strict evaluation:
 
 ```console
-$ nix-instantiate --eval --xml --expr 'rec { x = "foo"; y = x; }'
-...
-  <attr name="x">
-    <string value="foo" />
-  </attr>
-  <attr name="y">
-    <unevaluated />
-  </attr>
-...
+$ nix-instantiate --eval --xml --expr '{ x = {}; }'
+<?xml version='1.0' encoding='utf-8'?>
+<expr>
+  <attrs>
+    <attr column="3" line="1" name="x">
+      <unevaluated />
+    </attr>
+  </attrs>
+</expr>
 ```
 
 Note that `y` is left unevaluated (the XML representation doesn’t
 attempt to show non-normal forms).
 
 ```console
-$ nix-instantiate --eval --xml --strict --expr 'rec { x = "foo"; y = x; }'
-...
-  <attr name="x">
-    <string value="foo" />
-  </attr>
-  <attr name="y">
-    <string value="foo" />
-  </attr>
-...
+$ nix-instantiate --eval --xml --strict --expr '{ x = {}; }'
+<?xml version='1.0' encoding='utf-8'?>
+<expr>
+  <attrs>
+    <attr column="3" line="1" name="x">
+      <attrs>
+      </attrs>
+    </attr>
+  </attrs>
+</expr>
 ```
diff --git a/src/libexpr/print-ambiguous.cc b/src/libexpr/print-ambiguous.cc
new file mode 100644
index 000000000..07c398dd2
--- /dev/null
+++ b/src/libexpr/print-ambiguous.cc
@@ -0,0 +1,100 @@
+#include "print-ambiguous.hh"
+#include "print.hh"
+#include "signals.hh"
+
+namespace nix {
+
+// See: https://github.com/NixOS/nix/issues/9730
+void printAmbiguous(
+    Value &v,
+    const SymbolTable &symbols,
+    std::ostream &str,
+    std::set<const void *> *seen,
+    int depth)
+{
+    checkInterrupt();
+
+    if (depth <= 0) {
+        str << "«too deep»";
+        return;
+    }
+    switch (v.type()) {
+    case nInt:
+        str << v.integer;
+        break;
+    case nBool:
+        printLiteralBool(str, v.boolean);
+        break;
+    case nString:
+        printLiteralString(str, v.string_view());
+        break;
+    case nPath:
+        str << v.path().to_string(); // !!! escaping?
+        break;
+    case nNull:
+        str << "null";
+        break;
+    case nAttrs: {
+        if (seen && !v.attrs->empty() && !seen->insert(v.attrs).second)
+            str << "«repeated»";
+        else {
+            str << "{ ";
+            for (auto & i : v.attrs->lexicographicOrder(symbols)) {
+                str << symbols[i->name] << " = ";
+                printAmbiguous(*i->value, symbols, str, seen, depth - 1);
+                str << "; ";
+            }
+            str << "}";
+        }
+        break;
+    }
+    case nList:
+        if (seen && v.listSize() && !seen->insert(v.listElems()).second)
+            str << "«repeated»";
+        else {
+            str << "[ ";
+            for (auto v2 : v.listItems()) {
+                if (v2)
+                    printAmbiguous(*v2, symbols, str, seen, depth - 1);
+                else
+                    str << "(nullptr)";
+                str << " ";
+            }
+            str << "]";
+        }
+        break;
+    case nThunk:
+        if (!v.isBlackhole()) {
+            str << "<CODE>";
+        } else {
+            // Although we know for sure that it's going to be an infinite recursion
+            // when this value is accessed _in the current context_, it's likely
+            // that the user will misinterpret a simpler «infinite recursion» output
+            // as a definitive statement about the value, while in fact it may be
+            // a valid value after `builtins.trace` and perhaps some other steps
+            // have completed.
+            str << "«potential infinite recursion»";
+        }
+        break;
+    case nFunction:
+        if (v.isLambda()) {
+            str << "<LAMBDA>";
+        } else if (v.isPrimOp()) {
+            str << "<PRIMOP>";
+        } else if (v.isPrimOpApp()) {
+            str << "<PRIMOP-APP>";
+        }
+        break;
+    case nExternal:
+        str << *v.external;
+        break;
+    case nFloat:
+        str << v.fpoint;
+        break;
+    default:
+        printError("Nix evaluator internal error: printAmbiguous: invalid value type");
+        abort();
+    }
+}
+
+}
diff --git a/src/libexpr/print-ambiguous.hh b/src/libexpr/print-ambiguous.hh
new file mode 100644
index 000000000..50c260a9b
--- /dev/null
+++ b/src/libexpr/print-ambiguous.hh
@@ -0,0 +1,24 @@
+#pragma once
+
+#include "value.hh"
+
+namespace nix {
+
+/**
+ * Print a value in the deprecated format used by `nix-instantiate --eval` and
+ * `nix-env` (for manifests).
+ *
+ * This output can't be changed because it's part of the `nix-instantiate` API,
+ * but it produces ambiguous output; unevaluated thunks and lambdas (and a few
+ * other types) are printed as Nix path syntax like `<CODE>`.
+ *
+ * See: https://github.com/NixOS/nix/issues/9730
+ */
+void printAmbiguous(
+    Value &v,
+    const SymbolTable &symbols,
+    std::ostream &str,
+    std::set<const void *> *seen,
+    int depth);
+
+}
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index 3d07cab7a..973b6ee2b 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -108,8 +108,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
        environment. */
     auto manifestFile = ({
         std::ostringstream str;
-        std::set<const void *> seen;
-        printAmbiguous(manifest, state.symbols, str, &seen, std::numeric_limits<int>::max());
+        printAmbiguous(manifest, state.symbols, str, nullptr, std::numeric_limits<int>::max());
         // TODO with C++20 we can use str.view() instead and avoid copy.
         std::string str2 = str.str();
         StringSource source { str2 };
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index 9b36dccc6..87bc986e8 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -1,9 +1,11 @@
 #include "globals.hh"
+#include "print-ambiguous.hh"
 #include "shared.hh"
 #include "eval.hh"
 #include "eval-inline.hh"
 #include "get-drvs.hh"
 #include "attr-path.hh"
+#include "signals.hh"
 #include "value-to-xml.hh"
 #include "value-to-json.hh"
 #include "store-api.hh"
@@ -24,7 +26,6 @@ static int rootNr = 0;
 
 enum OutputKind { okPlain, okXML, okJSON };
 
-
 void processExpr(EvalState & state, const Strings & attrPaths,
     bool parseOnly, bool strict, Bindings & autoArgs,
     bool evalOnly, OutputKind output, bool location, Expr * e)
@@ -56,7 +57,8 @@ void processExpr(EvalState & state, const Strings & attrPaths,
                 std::cout << std::endl;
             } else {
                 if (strict) state.forceValueDeep(vRes);
-                vRes.print(state, std::cout);
+                std::set<const void *> seen;
+                printAmbiguous(vRes, state.symbols, std::cout, &seen, std::numeric_limits<int>::max());
                 std::cout << std::endl;
             }
         } else {
diff --git a/tests/functional/lang/eval-okay-print.exp b/tests/functional/lang/eval-okay-print.exp
index aa1b2379e..0d960fb70 100644
--- a/tests/functional/lang/eval-okay-print.exp
+++ b/tests/functional/lang/eval-okay-print.exp
@@ -1 +1 @@
-[ null «primop toString» «partially applied primop deepSeq» «lambda @ /pwd/lang/eval-okay-print.nix:1:61» [ [ «repeated» ] ] ]
+[ null <PRIMOP> <PRIMOP-APP> <LAMBDA> [ [ «repeated» ] ] ]

From 34bb6dcab1334ebc6ac0afaf4fe6f9e6f13de4b5 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 11 Jan 2024 11:14:13 -0500
Subject: [PATCH 334/421] makefiles: Support `.exe` executable prefix on
 Windows

---
 mk/programs.mk | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/mk/programs.mk b/mk/programs.mk
index 6235311e9..623caaf55 100644
--- a/mk/programs.mk
+++ b/mk/programs.mk
@@ -1,5 +1,11 @@
 programs-list :=
 
+ifdef HOST_WINDOWS
+  EXE_EXT = .exe
+else
+  EXE_EXT =
+endif
+
 # Build a program with symbolic name $(1).  The program is defined by
 # various variables prefixed by ‘$(1)_’:
 #
@@ -31,7 +37,7 @@ define build-program
   _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src)))
   $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs))))
   _libs := $$(foreach lib, $$($(1)_LIBS), $$(foreach lib2, $$($$(lib)_LIB_CLOSURE), $$($$(lib2)_PATH)))
-  $(1)_PATH := $$(_d)/$$($(1)_NAME)
+  $(1)_PATH := $$(_d)/$$($(1)_NAME)$(EXE_EXT)
 
   $$(eval $$(call create-dir, $$(_d)))
 
@@ -42,7 +48,7 @@ define build-program
 
   ifdef $(1)_INSTALL_DIR
 
-    $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$$($(1)_NAME)
+    $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$$($(1)_NAME)$(EXE_EXT)
 
     $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR)))
 

From af0345df3688494d1e53a659eacb16fc4b9915b5 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 11 Jan 2024 11:14:13 -0500
Subject: [PATCH 335/421] makefiles: Do some HOST_CYGWIN -> HOST_WINDOWS

These bits are not Cygwin-specific.
---
 mk/libraries.mk | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/mk/libraries.mk b/mk/libraries.mk
index 515a481f6..b99ba2782 100644
--- a/mk/libraries.mk
+++ b/mk/libraries.mk
@@ -3,7 +3,7 @@ libs-list :=
 ifdef HOST_DARWIN
   SO_EXT = dylib
 else
-  ifdef HOST_CYGWIN
+  ifdef HOST_WINDOWS
     SO_EXT = dll
   else
     SO_EXT = so
@@ -65,7 +65,7 @@ define build-library
   $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs))))
   _libs := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_PATH))
 
-  ifdef HOST_CYGWIN
+  ifdef HOST_WINDOWS
     $(1)_INSTALL_DIR ?= $$(bindir)
   else
     $(1)_INSTALL_DIR ?= $$(libdir)
@@ -85,7 +85,7 @@ define build-library
       endif
     else
       ifndef HOST_DARWIN
-        ifndef HOST_CYGWIN
+        ifndef HOST_WINDOWS
           $(1)_LDFLAGS += -Wl,-z,defs
         endif
       endif

From 90fdbfc601a8d005f57c984284c5922dc38480eb Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 9 Jan 2024 12:41:42 -0500
Subject: [PATCH 336/421] Build Windows DLLs with `-Wl,--export-all-symbols`

This is not the most elegant, but will match the SOs in exporting
everything for now. Later we can refine what is public/private to clean
up the interface.
---
 Makefile       | 37 ++++++++++++++++++++++++++++++++-----
 mk/lib.mk      | 33 +--------------------------------
 mk/platform.mk | 32 ++++++++++++++++++++++++++++++++
 3 files changed, 65 insertions(+), 37 deletions(-)
 create mode 100644 mk/platform.mk

diff --git a/Makefile b/Makefile
index 1fdb6e897..7bbfbddbe 100644
--- a/Makefile
+++ b/Makefile
@@ -1,8 +1,12 @@
+# External build directory support
+
 include mk/build-dir.mk
 
 -include $(buildprefix)Makefile.config
 clean-files += $(buildprefix)Makefile.config
 
+# List makefiles
+
 ifeq ($(ENABLE_BUILD), yes)
 makefiles = \
   mk/precompiled-headers.mk \
@@ -43,6 +47,8 @@ makefiles += \
   tests/functional/plugins/local.mk
 endif
 
+# Miscellaneous global Flags
+
 OPTIMIZE = 1
 
 ifeq ($(OPTIMIZE), 1)
@@ -52,9 +58,29 @@ else
   GLOBAL_CXXFLAGS += -O0 -U_FORTIFY_SOURCE
 endif
 
+include mk/platform.mk
+
+ifdef HOST_WINDOWS
+  # Windows DLLs are stricter about symbol visibility than Unix shared
+  # objects --- see https://gcc.gnu.org/wiki/Visibility for details.
+  # This is a temporary sledgehammer to export everything like on Unix,
+  # and not detail with this yet.
+  #
+  # TODO do not do this, and instead do fine-grained export annotations.
+  GLOBAL_LDFLAGS += -Wl,--export-all-symbols
+endif
+
+GLOBAL_CXXFLAGS += -g -Wall -include $(buildprefix)config.h -std=c++2a -I src
+
+# Include the main lib, causing rules to be defined
+
 include mk/lib.mk
 
-# Must be included after `mk/lib.mk` so isn't the default target.
+# Fallback stub rules for better UX when things are disabled
+#
+# These must be defined after `mk/lib.mk`. Otherwise the first rule
+# incorrectly becomes the default target.
+
 ifneq ($(ENABLE_UNIT_TESTS), yes)
 .PHONY: check
 check:
@@ -69,8 +95,11 @@ installcheck:
 	@exit 1
 endif
 
-# Must be included after `mk/lib.mk` so rules refer to variables defined
-# by the library. Rules are not "lazy" like variables, unfortunately.
+# Documentation or else fallback stub rules.
+#
+# The documentation makefiles be included after `mk/lib.mk` so rules
+# refer to variables defined by `mk/lib.mk`. Rules are not "lazy" like
+# variables, unfortunately.
 
 ifeq ($(ENABLE_DOC_GEN), yes)
 $(eval $(call include-sub-makefile, doc/manual/local.mk))
@@ -89,5 +118,3 @@ internal-api-html:
 	@echo "Internal API docs are disabled. Configure with '--enable-internal-api-docs', or avoid calling 'make internal-api-html'."
 	@exit 1
 endif
-
-GLOBAL_CXXFLAGS += -g -Wall -include $(buildprefix)config.h -std=c++2a -I src
diff --git a/mk/lib.mk b/mk/lib.mk
index a5a067e48..10ce8d436 100644
--- a/mk/lib.mk
+++ b/mk/lib.mk
@@ -12,38 +12,7 @@ man-pages :=
 install-tests :=
 install-tests-groups :=
 
-ifdef HOST_OS
-  HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS)))
-  ifeq ($(patsubst mingw%,,$(HOST_KERNEL)),)
-    HOST_MINGW = 1
-    HOST_WINDOWS = 1
-  endif
-  ifeq ($(HOST_KERNEL), cygwin)
-    HOST_CYGWIN = 1
-    HOST_WINDOWS = 1
-    HOST_UNIX = 1
-  endif
-  ifeq ($(patsubst darwin%,,$(HOST_KERNEL)),)
-    HOST_DARWIN = 1
-    HOST_UNIX = 1
-  endif
-  ifeq ($(patsubst freebsd%,,$(HOST_KERNEL)),)
-    HOST_FREEBSD = 1
-    HOST_UNIX = 1
-  endif
-  ifeq ($(patsubst netbsd%,,$(HOST_KERNEL)),)
-    HOST_NETBSD = 1
-    HOST_UNIX = 1
-  endif
-  ifeq ($(HOST_KERNEL), linux)
-    HOST_LINUX = 1
-    HOST_UNIX = 1
-  endif
-  ifeq ($(patsubst solaris%,,$(HOST_KERNEL)),)
-    HOST_SOLARIS = 1
-    HOST_UNIX = 1
-  endif
-endif
+include mk/platform.mk
 
 # Hack to define a literal space.
 space :=
diff --git a/mk/platform.mk b/mk/platform.mk
new file mode 100644
index 000000000..fe960dedf
--- /dev/null
+++ b/mk/platform.mk
@@ -0,0 +1,32 @@
+ifdef HOST_OS
+  HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS)))
+  ifeq ($(patsubst mingw%,,$(HOST_KERNEL)),)
+    HOST_MINGW = 1
+    HOST_WINDOWS = 1
+  endif
+  ifeq ($(HOST_KERNEL), cygwin)
+    HOST_CYGWIN = 1
+    HOST_WINDOWS = 1
+    HOST_UNIX = 1
+  endif
+  ifeq ($(patsubst darwin%,,$(HOST_KERNEL)),)
+    HOST_DARWIN = 1
+    HOST_UNIX = 1
+  endif
+  ifeq ($(patsubst freebsd%,,$(HOST_KERNEL)),)
+    HOST_FREEBSD = 1
+    HOST_UNIX = 1
+  endif
+  ifeq ($(patsubst netbsd%,,$(HOST_KERNEL)),)
+    HOST_NETBSD = 1
+    HOST_UNIX = 1
+  endif
+  ifeq ($(HOST_KERNEL), linux)
+    HOST_LINUX = 1
+    HOST_UNIX = 1
+  endif
+  ifeq ($(patsubst solaris%,,$(HOST_KERNEL)),)
+    HOST_SOLARIS = 1
+    HOST_UNIX = 1
+  endif
+endif

From 3e237598342dee46188c83fba49cc30d509ee553 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 12 Jan 2024 12:38:55 +0100
Subject: [PATCH 337/421] gc-non-blocking.sh: Add explanation

Also name the _NIX_TEST_GC_SYNC environment variables logically.
---
 src/libstore/gc.cc                  | 10 +++++-----
 tests/functional/gc-non-blocking.sh | 20 ++++++++++++++------
 2 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index bd64e238d..80e036e7e 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -511,7 +511,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
     /* Synchronisation point to test ENOENT handling in
        addTempRoot(), see tests/gc-non-blocking.sh. */
-    if (auto p = getEnv("_NIX_TEST_GC_SYNC_2"))
+    if (auto p = getEnv("_NIX_TEST_GC_SYNC_1"))
         readFile(*p);
 
     /* Start the server for receiving new roots. */
@@ -637,6 +637,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
         roots.insert(root.first);
     }
 
+    /* Synchronisation point for testing, see tests/functional/gc-non-blocking.sh. */
+    if (auto p = getEnv("_NIX_TEST_GC_SYNC_2"))
+        readFile(*p);
+
     /* Helper function that deletes a path from the store and throws
        GCLimitReached if we've deleted enough garbage. */
     auto deleteFromStore = [&](std::string_view baseName)
@@ -783,10 +787,6 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
         }
     };
 
-    /* Synchronisation point for testing, see tests/functional/gc-non-blocking.sh. */
-    if (auto p = getEnv("_NIX_TEST_GC_SYNC"))
-        readFile(*p);
-
     /* Either delete all garbage paths, or just the specified
        paths (for gcDeleteSpecific). */
     if (options.action == GCOptions::gcDeleteSpecific) {
diff --git a/tests/functional/gc-non-blocking.sh b/tests/functional/gc-non-blocking.sh
index 7f2aebb8b..ec280badb 100644
--- a/tests/functional/gc-non-blocking.sh
+++ b/tests/functional/gc-non-blocking.sh
@@ -6,10 +6,14 @@ needLocalStore "the GC test needs a synchronisation point"
 
 clearStore
 
-fifo=$TEST_ROOT/test.fifo
-mkfifo "$fifo"
+# This FIFO is read just after the global GC lock has been acquired,
+# but before the root server is started.
+fifo1=$TEST_ROOT/test2.fifo
+mkfifo "$fifo1"
 
-fifo2=$TEST_ROOT/test2.fifo
+# This FIFO is read just after the roots have been read, but before
+# the actual GC starts.
+fifo2=$TEST_ROOT/test.fifo
 mkfifo "$fifo2"
 
 dummy=$(nix store add-path ./simple.nix)
@@ -17,19 +21,23 @@ dummy=$(nix store add-path ./simple.nix)
 running=$TEST_ROOT/running
 touch $running
 
-(_NIX_TEST_GC_SYNC=$fifo _NIX_TEST_GC_SYNC_2=$fifo2 nix-store --gc -vvvvv; rm $running) &
+# Start GC.
+(_NIX_TEST_GC_SYNC_1=$fifo1 _NIX_TEST_GC_SYNC_2=$fifo2 nix-store --gc -vvvvv; rm $running) &
 pid=$!
 
 sleep 2
 
-(sleep 1; echo > $fifo2) &
+# Delay the start of the root server to check that the build below
+# correctly handles ENOENT when connecting to the root server.
+(sleep 1; echo > $fifo1) &
 pid2=$!
 
+# Start a build. This should not be blocked by the GC in progress.
 outPath=$(nix-build --max-silent-time 60 -o "$TEST_ROOT/result" -E "
   with import ./config.nix;
   mkDerivation {
     name = \"non-blocking\";
-    buildCommand = \"set -x; test -e $running; mkdir \$out; echo > $fifo\";
+    buildCommand = \"set -x; test -e $running; mkdir \$out; echo > $fifo2\";
   }")
 
 wait $pid

From 7c6f093abcb68a2d07cd6450672c120f33ab96d6 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 12 Jan 2024 13:00:53 +0100
Subject: [PATCH 338/421] .data() -> .c_str() to be on the safe side

---
 src/nix/develop.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 5e25833eb..1f2891378 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -603,7 +603,7 @@ struct CmdDevelop : Common, MixEnvironment
 
         setEnviron();
         // prevent garbage collection until shell exits
-        setenv("NIX_GCROOT", gcroot.data(), 1);
+        setenv("NIX_GCROOT", gcroot.c_str(), 1);
 
         Path shell = "bash";
 
@@ -648,7 +648,7 @@ struct CmdDevelop : Common, MixEnvironment
 
         // Override SHELL with the one chosen for this environment.
         // This is to make sure the system shell doesn't leak into the build environment.
-        setenv("SHELL", shell.data(), 1);
+        setenv("SHELL", shell.c_str(), 1);
 
         // If running a phase or single command, don't want an interactive shell running after
         // Ctrl-C, so don't pass --rcfile

From 4d0ecda33e29520756fdb7ccb7549205ed1afd52 Mon Sep 17 00:00:00 2001
From: DavHau <hsngrmpf+github@gmail.com>
Date: Sun, 19 Nov 2023 20:37:42 +0700
Subject: [PATCH 339/421] fetchTree/fetchGit: add test for .gitattributes

...with the intention to prevent future regressions in fetchGit
---
 tests/functional/fetchGit.sh | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh
index 4985c7764..f0438f548 100644
--- a/tests/functional/fetchGit.sh
+++ b/tests/functional/fetchGit.sh
@@ -229,6 +229,15 @@ rev_tag2=$(git -C $repo rev-parse refs/tags/tag2)
 [[ $rev_tag2_nix = $rev_tag2 ]]
 unset _NIX_FORCE_HTTP
 
+# Ensure .gitattributes is respected
+touch $repo/not-exported-file
+echo "/not-exported-file export-ignore" >> $repo/.gitattributes
+git -C $repo add not-exported-file .gitattributes
+git -C $repo commit -m 'Bla6'
+rev5=$(git -C $repo rev-parse HEAD)
+path12=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev5\"; }).outPath")
+[[ ! -e $path12/not-exported-file ]]
+
 # should fail if there is no repo
 rm -rf $repo/.git
 (! nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath")

From ce6d58a97cf6f975a0b930605605fab153445b22 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 27 Nov 2023 22:34:41 +0100
Subject: [PATCH 340/421] git fetcher: Add exportIgnore parameter

Enabled for fetchGit, which historically had this behavior,
among other behaviors we do not want in fetchGit.

fetchTree disables this parameter by default. It can choose the
simpler behavior, as it is still experimental.

I am not confident that the filtering implementation is future
proof. It should reuse a source filtering wrapper, which I believe
Eelco has already written, but not merged yet.
---
 src/libexpr/primops/fetchTree.cc | 14 ++++++++
 src/libfetchers/git-utils.cc     | 57 +++++++++++++++++++++++++++-----
 src/libfetchers/git-utils.hh     |  4 +--
 src/libfetchers/git.cc           | 15 +++++++--
 tests/functional/fetchGit.sh     |  5 ++-
 5 files changed, 81 insertions(+), 14 deletions(-)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index eb2df8626..e00c4f190 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -39,6 +39,10 @@ void emitTreeAttrs(
         attrs.alloc("submodules").mkBool(
             fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
 
+    if (input.getType() == "git")
+        attrs.alloc("exportIgnore").mkBool(
+            fetchers::maybeGetBoolAttr(input.attrs, "exportIgnore").value_or(false));
+
     if (!forceDirty) {
 
         if (auto rev = input.getRev()) {
@@ -112,6 +116,11 @@ static void fetchTree(
 
         attrs.emplace("type", type.value());
 
+        if (params.isFetchGit) {
+            // Default value; user attrs are assigned later.
+            attrs.emplace("exportIgnore", Explicit<bool>{true});
+        }
+
         for (auto & attr : *args[0]->attrs) {
             if (attr.name == state.sType) continue;
             state.forceValue(*attr.value, attr.pos);
@@ -593,6 +602,11 @@ static RegisterPrimOp primop_fetchGit({
 
         A Boolean parameter that specifies whether submodules should be checked out.
 
+      - `exportIgnore` (default: `true`)
+
+        A Boolean parameter that specifies whether `export-ignore` from `.gitattributes` should be applied.
+        This approximates part of the `git archive` behavior.
+
       - `shallow` (default: `false`)
 
         A Boolean parameter that specifies whether fetching from a shallow remote repository is allowed.
diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 65f7b45ef..4dc749504 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -7,6 +7,7 @@
 
 #include <boost/core/span.hpp>
 
+#include <git2/attr.h>
 #include <git2/blob.h>
 #include <git2/commit.h>
 #include <git2/config.h>
@@ -21,6 +22,7 @@
 #include <git2/submodule.h>
 #include <git2/tree.h>
 
+#include <iostream>
 #include <unordered_set>
 #include <queue>
 #include <regex>
@@ -307,7 +309,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         return std::nullopt;
     }
 
-    std::vector<std::tuple<Submodule, Hash>> getSubmodules(const Hash & rev) override;
+    std::vector<std::tuple<Submodule, Hash>> getSubmodules(const Hash & rev, bool exportIgnore) override;
 
     std::string resolveSubmoduleUrl(
         const std::string & url,
@@ -340,7 +342,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         return true;
     }
 
-    ref<InputAccessor> getAccessor(const Hash & rev) override;
+    ref<InputAccessor> getAccessor(const Hash & rev, bool exportIgnore) override;
 
     static int sidebandProgressCallback(const char * str, int len, void * payload)
     {
@@ -460,10 +462,12 @@ struct GitInputAccessor : InputAccessor
 {
     ref<GitRepoImpl> repo;
     Tree root;
+    bool exportIgnore;
 
-    GitInputAccessor(ref<GitRepoImpl> repo_, const Hash & rev)
+    GitInputAccessor(ref<GitRepoImpl> repo_, const Hash & rev, bool exportIgnore)
         : repo(repo_)
         , root(peelObject<Tree>(*repo, lookupObject(*repo, hashToOID(rev)).get(), GIT_OBJECT_TREE))
+        , exportIgnore(exportIgnore)
     {
     }
 
@@ -492,7 +496,7 @@ struct GitInputAccessor : InputAccessor
             return Stat { .type = tDirectory };
 
         auto entry = lookup(path);
-        if (!entry)
+        if (!entry || isExportIgnored(path))
             return std::nullopt;
 
         auto mode = git_tree_entry_filemode(entry);
@@ -527,6 +531,12 @@ struct GitInputAccessor : InputAccessor
 
                 for (size_t n = 0; n < count; ++n) {
                     auto entry = git_tree_entry_byindex(tree.get(), n);
+                    if (exportIgnore) {
+                        if (isExportIgnored(path + git_tree_entry_name(entry))) {
+                            continue;
+                        }
+                    }
+
                     // FIXME: add to cache
                     res.emplace(std::string(git_tree_entry_name(entry)), DirEntry{});
                 }
@@ -556,6 +566,33 @@ struct GitInputAccessor : InputAccessor
 
     std::unordered_map<CanonPath, TreeEntry> lookupCache;
 
+    bool isExportIgnored(const CanonPath & path) {
+        if (!exportIgnore)
+            return false;
+
+        const char *exportIgnoreEntry = nullptr;
+
+        // GIT_ATTR_CHECK_INDEX_ONLY:
+        // > It will use index only for creating archives or for a bare repo
+        // > (if an index has been specified for the bare repo).
+        // -- https://github.com/libgit2/libgit2/blob/HEAD/include/git2/attr.h#L113C62-L115C48
+        if (git_attr_get(&exportIgnoreEntry,
+                *repo,
+                GIT_ATTR_CHECK_INDEX_ONLY,
+                std::string(path.rel()).c_str(),
+                "export-ignore")) {
+            if (git_error_last()->klass == GIT_ENOTFOUND)
+                return false;
+            else
+                throw Error("looking up '%s': %s", showPath(path), git_error_last()->message);
+        }
+        else {
+            // Official git will silently reject export-ignore lines that have
+            // values. We do the same.
+            return GIT_ATTR_IS_TRUE(exportIgnoreEntry);
+        }
+    }
+
     /* Recursively look up 'path' relative to the root. */
     git_tree_entry * lookup(const CanonPath & path)
     {
@@ -569,6 +606,10 @@ struct GitInputAccessor : InputAccessor
                     throw Error("looking up '%s': %s", showPath(path), git_error_last()->message);
             }
 
+            if (entry && isExportIgnored(path)) {
+                entry.reset();
+            }
+
             i = lookupCache.emplace(path, std::move(entry)).first;
         }
 
@@ -644,17 +685,17 @@ struct GitInputAccessor : InputAccessor
     }
 };
 
-ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev)
+ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore)
 {
-    return make_ref<GitInputAccessor>(ref<GitRepoImpl>(shared_from_this()), rev);
+    return make_ref<GitInputAccessor>(ref<GitRepoImpl>(shared_from_this()), rev, exportIgnore);
 }
 
-std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules(const Hash & rev)
+std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules(const Hash & rev, bool exportIgnore)
 {
     /* Read the .gitmodules files from this revision. */
     CanonPath modulesFile(".gitmodules");
 
-    auto accessor = getAccessor(rev);
+    auto accessor = getAccessor(rev, exportIgnore);
     if (!accessor->pathExists(modulesFile)) return {};
 
     /* Parse it and get the revision of each submodule. */
diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh
index 1def82071..f1cb48065 100644
--- a/src/libfetchers/git-utils.hh
+++ b/src/libfetchers/git-utils.hh
@@ -57,7 +57,7 @@ struct GitRepo
      * Return the submodules of this repo at the indicated revision,
      * along with the revision of each submodule.
      */
-    virtual std::vector<std::tuple<Submodule, Hash>> getSubmodules(const Hash & rev) = 0;
+    virtual std::vector<std::tuple<Submodule, Hash>> getSubmodules(const Hash & rev, bool exportIgnore) = 0;
 
     virtual std::string resolveSubmoduleUrl(
         const std::string & url,
@@ -71,7 +71,7 @@ struct GitRepo
 
     virtual bool hasObject(const Hash & oid) = 0;
 
-    virtual ref<InputAccessor> getAccessor(const Hash & rev) = 0;
+    virtual ref<InputAccessor> getAccessor(const Hash & rev, bool exportIgnore) = 0;
 
     virtual void fetch(
         const std::string & url,
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 79270c317..fb8bf5bf4 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -174,7 +174,7 @@ struct GitInputScheme : InputScheme
         for (auto & [name, value] : url.query) {
             if (name == "rev" || name == "ref" || name == "keytype" || name == "publicKey" || name == "publicKeys")
                 attrs.emplace(name, value);
-            else if (name == "shallow" || name == "submodules" || name == "allRefs" || name == "verifyCommit")
+            else if (name == "shallow" || name == "submodules" || name == "exportIgnore" || name == "allRefs" || name == "verifyCommit")
                 attrs.emplace(name, Explicit<bool> { value == "1" });
             else
                 url2.query.emplace(name, value);
@@ -199,6 +199,7 @@ struct GitInputScheme : InputScheme
             "rev",
             "shallow",
             "submodules",
+            "exportIgnore",
             "lastModified",
             "revCount",
             "narHash",
@@ -250,6 +251,8 @@ struct GitInputScheme : InputScheme
             url.query.insert_or_assign("shallow", "1");
         if (getSubmodulesAttr(input))
             url.query.insert_or_assign("submodules", "1");
+        if (maybeGetBoolAttr(input.attrs, "exportIgnore").value_or(false))
+            url.query.insert_or_assign("exportIgnore", "1");
         if (maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(false))
             url.query.insert_or_assign("verifyCommit", "1");
         auto publicKeys = getPublicKeys(input.attrs);
@@ -372,6 +375,11 @@ struct GitInputScheme : InputScheme
         return maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
     }
 
+    bool getExportIgnoreAttr(const Input & input) const
+    {
+        return maybeGetBoolAttr(input.attrs, "exportIgnore").value_or(false);
+    }
+
     bool getAllRefsAttr(const Input & input) const
     {
         return maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
@@ -600,7 +608,8 @@ struct GitInputScheme : InputScheme
 
         verifyCommit(input, repo);
 
-        auto accessor = repo->getAccessor(rev);
+        bool exportIgnore = getExportIgnoreAttr(input);
+        auto accessor = repo->getAccessor(rev, exportIgnore);
 
         accessor->setPathDisplay("«" + input.to_string() + "»");
 
@@ -610,7 +619,7 @@ struct GitInputScheme : InputScheme
         if (getSubmodulesAttr(input)) {
             std::map<CanonPath, nix::ref<InputAccessor>> mounts;
 
-            for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev)) {
+            for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev, exportIgnore)) {
                 auto resolved = repo->resolveSubmoduleUrl(submodule.url, repoInfo.url);
                 debug("Git submodule %s: %s %s %s -> %s",
                     submodule.path, submodule.url, submodule.branch, submoduleRev.gitRev(), resolved);
diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh
index f0438f548..46532c025 100644
--- a/tests/functional/fetchGit.sh
+++ b/tests/functional/fetchGit.sh
@@ -231,12 +231,15 @@ unset _NIX_FORCE_HTTP
 
 # Ensure .gitattributes is respected
 touch $repo/not-exported-file
+touch $repo/exported-wonky
 echo "/not-exported-file export-ignore" >> $repo/.gitattributes
-git -C $repo add not-exported-file .gitattributes
+echo "/exported-wonky export-ignore=wonk" >> $repo/.gitattributes
+git -C $repo add not-exported-file exported-wonky .gitattributes
 git -C $repo commit -m 'Bla6'
 rev5=$(git -C $repo rev-parse HEAD)
 path12=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev5\"; }).outPath")
 [[ ! -e $path12/not-exported-file ]]
+[[ -e $path12/exported-wonky ]]
 
 # should fail if there is no repo
 rm -rf $repo/.git

From 1c6bb609af3277ff3f747f49d04be80463d1f153 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Tue, 28 Nov 2023 00:41:01 +0100
Subject: [PATCH 341/421] fetchTree: allow larger output attrsets

Intentionally dumb change ahead of architectural improvements.
---
 src/libexpr/primops/fetchTree.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index e00c4f190..d04908b77 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -25,7 +25,7 @@ void emitTreeAttrs(
 {
     assert(input.isLocked());
 
-    auto attrs = state.buildBindings(10);
+    auto attrs = state.buildBindings(100);
 
     state.mkStorePathString(storePath, attrs.alloc(state.sOutPath));
 

From f6b1d155804a946ff2965b5fd1a57159770e8b58 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 11 Dec 2023 19:27:13 +0100
Subject: [PATCH 342/421] MakeNotAllowedError: Touch up doc

---
 src/libfetchers/filtering-input-accessor.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libfetchers/filtering-input-accessor.hh b/src/libfetchers/filtering-input-accessor.hh
index a352a33a6..2e2495c78 100644
--- a/src/libfetchers/filtering-input-accessor.hh
+++ b/src/libfetchers/filtering-input-accessor.hh
@@ -6,7 +6,7 @@
 namespace nix {
 
 /**
- * A function that should throw an exception of type
+ * A function that returns an exception of type
  * `RestrictedPathError` explaining that access to `path` is
  * forbidden.
  */

From cd5e752fa72bf15ba8fe6fcdae92c77ac6dc2375 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 11 Dec 2023 19:30:10 +0100
Subject: [PATCH 343/421] GitRepoImpl::getSubmodules: Access getSubmoduleRev
 without cast

This will be needed because the accessor will be wrapped, and therefore
not be an instance of GitInputAccessor anymore.
---
 src/libfetchers/git-utils.cc | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 4dc749504..d8a4f1778 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -52,6 +52,8 @@ bool operator == (const git_oid & oid1, const git_oid & oid2)
 
 namespace nix {
 
+struct GitInputAccessor;
+
 // Some wrapper types that ensure that the git_*_free functions get called.
 template<auto del>
 struct Deleter
@@ -342,6 +344,11 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         return true;
     }
 
+    /**
+     * A 'GitInputAccessor' with no regard for export-ignore or any other transformations.
+     */
+    ref<GitInputAccessor> getRawAccessor(const Hash & rev);
+
     ref<InputAccessor> getAccessor(const Hash & rev, bool exportIgnore) override;
 
     static int sidebandProgressCallback(const char * str, int len, void * payload)
@@ -685,6 +692,12 @@ struct GitInputAccessor : InputAccessor
     }
 };
 
+ref<GitInputAccessor> GitRepoImpl::getRawAccessor(const Hash & rev)
+{
+    auto self = ref<GitRepoImpl>(shared_from_this());
+    return make_ref<GitInputAccessor>(self, rev);
+}
+
 ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore)
 {
     return make_ref<GitInputAccessor>(ref<GitRepoImpl>(shared_from_this()), rev, exportIgnore);
@@ -706,8 +719,10 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
 
     std::vector<std::tuple<Submodule, Hash>> result;
 
+    auto rawAccessor = getRawAccessor(rev);
+
     for (auto & submodule : parseSubmodules(CanonPath(pathTemp))) {
-        auto rev = accessor.dynamic_pointer_cast<GitInputAccessor>()->getSubmoduleRev(submodule.path);
+        auto rev = rawAccessor->getSubmoduleRev(submodule.path);
         result.push_back({std::move(submodule), rev});
     }
 

From 467c62a96eaabe2f71939a07d923a759f82a466f Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 11 Dec 2023 19:32:18 +0100
Subject: [PATCH 344/421] GitRepoImpl: Move exportIgnore into a filtering
 accessor

---
 src/libfetchers/git-utils.cc | 96 ++++++++++++++++++++----------------
 1 file changed, 54 insertions(+), 42 deletions(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index d8a4f1778..f8b2afeef 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -1,5 +1,6 @@
 #include "git-utils.hh"
 #include "input-accessor.hh"
+#include "filtering-input-accessor.hh"
 #include "cache.hh"
 #include "finally.hh"
 #include "processes.hh"
@@ -465,16 +466,17 @@ ref<GitRepo> GitRepo::openRepo(const CanonPath & path, bool create, bool bare)
     return make_ref<GitRepoImpl>(path, create, bare);
 }
 
+/**
+ * Raw git tree input accessor.
+ */
 struct GitInputAccessor : InputAccessor
 {
     ref<GitRepoImpl> repo;
     Tree root;
-    bool exportIgnore;
 
-    GitInputAccessor(ref<GitRepoImpl> repo_, const Hash & rev, bool exportIgnore)
+    GitInputAccessor(ref<GitRepoImpl> repo_, const Hash & rev)
         : repo(repo_)
         , root(peelObject<Tree>(*repo, lookupObject(*repo, hashToOID(rev)).get(), GIT_OBJECT_TREE))
-        , exportIgnore(exportIgnore)
     {
     }
 
@@ -503,7 +505,7 @@ struct GitInputAccessor : InputAccessor
             return Stat { .type = tDirectory };
 
         auto entry = lookup(path);
-        if (!entry || isExportIgnored(path))
+        if (!entry)
             return std::nullopt;
 
         auto mode = git_tree_entry_filemode(entry);
@@ -538,12 +540,6 @@ struct GitInputAccessor : InputAccessor
 
                 for (size_t n = 0; n < count; ++n) {
                     auto entry = git_tree_entry_byindex(tree.get(), n);
-                    if (exportIgnore) {
-                        if (isExportIgnored(path + git_tree_entry_name(entry))) {
-                            continue;
-                        }
-                    }
-
                     // FIXME: add to cache
                     res.emplace(std::string(git_tree_entry_name(entry)), DirEntry{});
                 }
@@ -573,33 +569,6 @@ struct GitInputAccessor : InputAccessor
 
     std::unordered_map<CanonPath, TreeEntry> lookupCache;
 
-    bool isExportIgnored(const CanonPath & path) {
-        if (!exportIgnore)
-            return false;
-
-        const char *exportIgnoreEntry = nullptr;
-
-        // GIT_ATTR_CHECK_INDEX_ONLY:
-        // > It will use index only for creating archives or for a bare repo
-        // > (if an index has been specified for the bare repo).
-        // -- https://github.com/libgit2/libgit2/blob/HEAD/include/git2/attr.h#L113C62-L115C48
-        if (git_attr_get(&exportIgnoreEntry,
-                *repo,
-                GIT_ATTR_CHECK_INDEX_ONLY,
-                std::string(path.rel()).c_str(),
-                "export-ignore")) {
-            if (git_error_last()->klass == GIT_ENOTFOUND)
-                return false;
-            else
-                throw Error("looking up '%s': %s", showPath(path), git_error_last()->message);
-        }
-        else {
-            // Official git will silently reject export-ignore lines that have
-            // values. We do the same.
-            return GIT_ATTR_IS_TRUE(exportIgnoreEntry);
-        }
-    }
-
     /* Recursively look up 'path' relative to the root. */
     git_tree_entry * lookup(const CanonPath & path)
     {
@@ -613,10 +582,6 @@ struct GitInputAccessor : InputAccessor
                     throw Error("looking up '%s': %s", showPath(path), git_error_last()->message);
             }
 
-            if (entry && isExportIgnored(path)) {
-                entry.reset();
-            }
-
             i = lookupCache.emplace(path, std::move(entry)).first;
         }
 
@@ -692,6 +657,46 @@ struct GitInputAccessor : InputAccessor
     }
 };
 
+struct GitExportIgnoreInputAccessor : FilteringInputAccessor {
+    ref<GitRepoImpl> repo;
+
+    GitExportIgnoreInputAccessor(ref<GitRepoImpl> repo, ref<InputAccessor> next)
+        : FilteringInputAccessor(next, [&](const CanonPath & path) {
+            return RestrictedPathError(fmt("'%s' does not exist because it was fetched with exportIgnore enabled", path));
+        })
+        , repo(repo)
+    { }
+
+    bool isExportIgnored(const CanonPath & path) {
+        const char *exportIgnoreEntry = nullptr;
+
+        // GIT_ATTR_CHECK_INDEX_ONLY:
+        // > It will use index only for creating archives or for a bare repo
+        // > (if an index has been specified for the bare repo).
+        // -- https://github.com/libgit2/libgit2/blob/HEAD/include/git2/attr.h#L113C62-L115C48
+        if (git_attr_get(&exportIgnoreEntry,
+                *repo,
+                GIT_ATTR_CHECK_INDEX_ONLY,
+                std::string(path.rel()).c_str(),
+                "export-ignore")) {
+            if (git_error_last()->klass == GIT_ENOTFOUND)
+                return false;
+            else
+                throw Error("looking up '%s': %s", showPath(path), git_error_last()->message);
+        }
+        else {
+            // Official git will silently reject export-ignore lines that have
+            // values. We do the same.
+            return GIT_ATTR_IS_TRUE(exportIgnoreEntry);
+        }
+    }
+
+    bool isAllowed(const CanonPath & path) override {
+        return !isExportIgnored(path);
+    }
+
+};
+
 ref<GitInputAccessor> GitRepoImpl::getRawAccessor(const Hash & rev)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
@@ -700,7 +705,14 @@ ref<GitInputAccessor> GitRepoImpl::getRawAccessor(const Hash & rev)
 
 ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore)
 {
-    return make_ref<GitInputAccessor>(ref<GitRepoImpl>(shared_from_this()), rev, exportIgnore);
+    auto self = ref<GitRepoImpl>(shared_from_this());
+    ref<GitInputAccessor> rawGitAccessor = getRawAccessor(rev);
+    if (exportIgnore) {
+        return make_ref<GitExportIgnoreInputAccessor>(self, rawGitAccessor);
+    }
+    else {
+        return rawGitAccessor;
+    }
 }
 
 std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules(const Hash & rev, bool exportIgnore)

From 8024b954d702e0693b532650230037e398453693 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 11 Dec 2023 19:42:46 +0100
Subject: [PATCH 345/421] fetchTree: Recommend against exportIgnore

---
 src/libexpr/primops/fetchTree.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index d04908b77..2e4b72c9f 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -607,6 +607,8 @@ static RegisterPrimOp primop_fetchGit({
         A Boolean parameter that specifies whether `export-ignore` from `.gitattributes` should be applied.
         This approximates part of the `git archive` behavior.
 
+        Enabling this option is not recommended because it is unknown whether the Git developers commit to the reproducibility of `export-ignore` in newer Git versions.
+
       - `shallow` (default: `false`)
 
         A Boolean parameter that specifies whether fetching from a shallow remote repository is allowed.

From 7774eff10e0ec1f540a6dc22d8fd78de40714bdf Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 11 Dec 2023 22:28:53 +0100
Subject: [PATCH 346/421] libfetchers/git: Move workdir accessor into
 GitRepo::getAccessor

---
 src/libfetchers/git-utils.cc | 19 +++++++++++++++++++
 src/libfetchers/git-utils.hh |  3 +++
 src/libfetchers/git.cc       |  8 ++++----
 3 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index f8b2afeef..d218276b4 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -1,4 +1,5 @@
 #include "git-utils.hh"
+#include "fs-input-accessor.hh"
 #include "input-accessor.hh"
 #include "filtering-input-accessor.hh"
 #include "cache.hh"
@@ -352,6 +353,8 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
     ref<InputAccessor> getAccessor(const Hash & rev, bool exportIgnore) override;
 
+    ref<InputAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError e) override;
+
     static int sidebandProgressCallback(const char * str, int len, void * payload)
     {
         auto act = (Activity *) payload;
@@ -715,6 +718,22 @@ ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore)
     }
 }
 
+ref<InputAccessor> GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError)
+{
+    auto self = ref<GitRepoImpl>(shared_from_this());
+    ref<InputAccessor> fileAccessor =
+        AllowListInputAccessor::create(
+                makeFSInputAccessor(path),
+                std::set<CanonPath> { wd.files },
+                std::move(makeNotAllowedError));
+    if (exportIgnore) {
+        return make_ref<GitExportIgnoreInputAccessor>(self, fileAccessor);
+    }
+    else {
+        return fileAccessor;
+    }
+}
+
 std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules(const Hash & rev, bool exportIgnore)
 {
     /* Read the .gitmodules files from this revision. */
diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh
index f1cb48065..768554780 100644
--- a/src/libfetchers/git-utils.hh
+++ b/src/libfetchers/git-utils.hh
@@ -1,5 +1,6 @@
 #pragma once
 
+#include "filtering-input-accessor.hh"
 #include "input-accessor.hh"
 
 namespace nix {
@@ -73,6 +74,8 @@ struct GitRepo
 
     virtual ref<InputAccessor> getAccessor(const Hash & rev, bool exportIgnore) = 0;
 
+    virtual ref<InputAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) = 0;
+
     virtual void fetch(
         const std::string & url,
         const std::string & refspec,
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index fb8bf5bf4..d7818988f 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -9,7 +9,6 @@
 #include "processes.hh"
 #include "git.hh"
 #include "fs-input-accessor.hh"
-#include "filtering-input-accessor.hh"
 #include "mounted-input-accessor.hh"
 #include "git-utils.hh"
 #include "logging.hh"
@@ -659,10 +658,11 @@ struct GitInputScheme : InputScheme
             for (auto & submodule : repoInfo.workdirInfo.submodules)
                 repoInfo.workdirInfo.files.insert(submodule.path);
 
+        auto repo = GitRepo::openRepo(CanonPath(repoInfo.url), false, false);
+
         ref<InputAccessor> accessor =
-            AllowListInputAccessor::create(
-                makeFSInputAccessor(CanonPath(repoInfo.url)),
-                std::move(repoInfo.workdirInfo.files),
+            repo->getAccessor(repoInfo.workdirInfo,
+                getExportIgnoreAttr(input),
                 makeNotAllowedError(repoInfo.url));
 
         /* If the repo has submodules, return a mounted input accessor

From 1bbe8371849f33da4edba23289de7b7e3c5d6c83 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 11 Dec 2023 22:35:11 +0100
Subject: [PATCH 347/421] fetchTree: Add isFetchGit exportIgnore

---
 src/libexpr/primops/fetchTree.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 2e4b72c9f..c167444b0 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -161,6 +161,7 @@ static void fetchTree(
             fetchers::Attrs attrs;
             attrs.emplace("type", "git");
             attrs.emplace("url", fixGitURL(url));
+            attrs.emplace("exportIgnore", Explicit<bool>{true});
             input = fetchers::Input::fromAttrs(std::move(attrs));
         } else {
             if (!experimentalFeatureSettings.isEnabled(Xp::Flakes))

From 99bd12f0b18b1a2a94639134c49c478c9ab56b3b Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 11 Dec 2023 22:36:08 +0100
Subject: [PATCH 348/421] fetchGit/fetchTree: Improve exportIgnore, submodule
 interaction

Also fingerprint and some preparatory improvements.

Testing is still not up to scratch because lots of logic is duplicated
between the workdir and commit cases.
---
 src/libexpr/primops/fetchTree.cc       | 16 ++++++----
 src/libfetchers/fetchers.hh            |  7 +++++
 src/libfetchers/git-utils.cc           | 43 +++++++++++++++++++++-----
 src/libfetchers/git.cc                 |  9 ++++--
 tests/functional/fetchGitSubmodules.sh | 42 +++++++++++++++++++++++++
 5 files changed, 101 insertions(+), 16 deletions(-)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index c167444b0..7a4725334 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -116,11 +116,6 @@ static void fetchTree(
 
         attrs.emplace("type", type.value());
 
-        if (params.isFetchGit) {
-            // Default value; user attrs are assigned later.
-            attrs.emplace("exportIgnore", Explicit<bool>{true});
-        }
-
         for (auto & attr : *args[0]->attrs) {
             if (attr.name == state.sType) continue;
             state.forceValue(*attr.value, attr.pos);
@@ -144,6 +139,12 @@ static void fetchTree(
                     state.symbols[attr.name], showType(*attr.value)));
         }
 
+        if (params.isFetchGit && !attrs.contains("exportIgnore")) {
+            // Default value; user attrs are assigned later.
+            // FIXME: exportIgnore := !submodules
+            attrs.emplace("exportIgnore", Explicit<bool>{true});
+        }
+
         if (!params.allowNameArgument)
             if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
                 state.debugThrowLastTrace(EvalError({
@@ -161,7 +162,10 @@ static void fetchTree(
             fetchers::Attrs attrs;
             attrs.emplace("type", "git");
             attrs.emplace("url", fixGitURL(url));
-            attrs.emplace("exportIgnore", Explicit<bool>{true});
+            if (!attrs.contains("exportIgnore")) {
+                // FIXME: exportIgnore := !submodules
+                attrs.emplace("exportIgnore", Explicit<bool>{true});
+            }
             input = fetchers::Input::fromAttrs(std::move(attrs));
         } else {
             if (!experimentalFeatureSettings.isEnabled(Xp::Flakes))
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index 5f3254b6d..036647830 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -187,6 +187,13 @@ struct InputScheme
     virtual bool isDirect(const Input & input) const
     { return true; }
 
+    /**
+     * A sufficiently unique string that can be used as a cache key to identify the `input`.
+     *
+     * Only known-equivalent inputs should return the same fingerprint.
+     *
+     * This is not a stable identifier between Nix versions, but not guaranteed to change either.
+     */
     virtual std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const
     { return std::nullopt; }
 };
diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index d218276b4..cd65e0fda 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -662,14 +662,45 @@ struct GitInputAccessor : InputAccessor
 
 struct GitExportIgnoreInputAccessor : FilteringInputAccessor {
     ref<GitRepoImpl> repo;
+    std::optional<Hash> rev;
 
-    GitExportIgnoreInputAccessor(ref<GitRepoImpl> repo, ref<InputAccessor> next)
+    GitExportIgnoreInputAccessor(ref<GitRepoImpl> repo, ref<InputAccessor> next, std::optional<Hash> rev)
         : FilteringInputAccessor(next, [&](const CanonPath & path) {
             return RestrictedPathError(fmt("'%s' does not exist because it was fetched with exportIgnore enabled", path));
         })
         , repo(repo)
+        , rev(rev)
     { }
 
+    bool gitAttrGet(const CanonPath & path, const char * attrName, const char * & valueOut)
+    {
+        std::string pathStr {path.rel()};
+        const char * pathCStr = pathStr.c_str();
+
+        if (rev) {
+            git_attr_options opts = GIT_ATTR_OPTIONS_INIT;
+            opts.attr_commit_id = hashToOID(*rev);
+            // TODO: test that gitattributes from global and system are not used
+            //       (ie more or less: home and etc - both of them!)
+            opts.flags = GIT_ATTR_CHECK_INCLUDE_COMMIT | GIT_ATTR_CHECK_NO_SYSTEM;
+            return git_attr_get_ext(
+                &valueOut,
+                *repo,
+                &opts,
+                pathCStr,
+                attrName
+                );
+        }
+        else {
+            return git_attr_get(
+                &valueOut,
+                *repo,
+                GIT_ATTR_CHECK_INDEX_ONLY | GIT_ATTR_CHECK_NO_SYSTEM,
+                pathCStr,
+                attrName);
+        }
+    }
+
     bool isExportIgnored(const CanonPath & path) {
         const char *exportIgnoreEntry = nullptr;
 
@@ -677,11 +708,7 @@ struct GitExportIgnoreInputAccessor : FilteringInputAccessor {
         // > It will use index only for creating archives or for a bare repo
         // > (if an index has been specified for the bare repo).
         // -- https://github.com/libgit2/libgit2/blob/HEAD/include/git2/attr.h#L113C62-L115C48
-        if (git_attr_get(&exportIgnoreEntry,
-                *repo,
-                GIT_ATTR_CHECK_INDEX_ONLY,
-                std::string(path.rel()).c_str(),
-                "export-ignore")) {
+        if (gitAttrGet(path, "export-ignore", exportIgnoreEntry)) {
             if (git_error_last()->klass == GIT_ENOTFOUND)
                 return false;
             else
@@ -711,7 +738,7 @@ ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore)
     auto self = ref<GitRepoImpl>(shared_from_this());
     ref<GitInputAccessor> rawGitAccessor = getRawAccessor(rev);
     if (exportIgnore) {
-        return make_ref<GitExportIgnoreInputAccessor>(self, rawGitAccessor);
+        return make_ref<GitExportIgnoreInputAccessor>(self, rawGitAccessor, rev);
     }
     else {
         return rawGitAccessor;
@@ -727,7 +754,7 @@ ref<InputAccessor> GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportI
                 std::set<CanonPath> { wd.files },
                 std::move(makeNotAllowedError));
     if (exportIgnore) {
-        return make_ref<GitExportIgnoreInputAccessor>(self, fileAccessor);
+        return make_ref<GitExportIgnoreInputAccessor>(self, fileAccessor, std::nullopt);
     }
     else {
         return fileAccessor;
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index d7818988f..10c0aef97 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -628,6 +628,7 @@ struct GitInputScheme : InputScheme
                 if (submodule.branch != "")
                     attrs.insert_or_assign("ref", submodule.branch);
                 attrs.insert_or_assign("rev", submoduleRev.gitRev());
+                attrs.insert_or_assign("exportIgnore", Explicit<bool>{ exportIgnore });
                 auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
                 auto [submoduleAccessor, submoduleInput2] =
                     submoduleInput.getAccessor(store);
@@ -660,9 +661,11 @@ struct GitInputScheme : InputScheme
 
         auto repo = GitRepo::openRepo(CanonPath(repoInfo.url), false, false);
 
+        auto exportIgnore = getExportIgnoreAttr(input);
+
         ref<InputAccessor> accessor =
             repo->getAccessor(repoInfo.workdirInfo,
-                getExportIgnoreAttr(input),
+                exportIgnore,
                 makeNotAllowedError(repoInfo.url));
 
         /* If the repo has submodules, return a mounted input accessor
@@ -676,6 +679,8 @@ struct GitInputScheme : InputScheme
                 fetchers::Attrs attrs;
                 attrs.insert_or_assign("type", "git");
                 attrs.insert_or_assign("url", submodulePath.abs());
+                attrs.insert_or_assign("exportIgnore", Explicit<bool>{ exportIgnore });
+
                 auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
                 auto [submoduleAccessor, submoduleInput2] =
                     submoduleInput.getAccessor(store);
@@ -747,7 +752,7 @@ struct GitInputScheme : InputScheme
     std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
     {
         if (auto rev = input.getRev())
-            return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : "");
+            return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "");
         else
             return std::nullopt;
     }
diff --git a/tests/functional/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh
index 369cdc5db..1b425820e 100644
--- a/tests/functional/fetchGitSubmodules.sh
+++ b/tests/functional/fetchGitSubmodules.sh
@@ -118,3 +118,45 @@ cloneRepo=$TEST_ROOT/a/b/gitSubmodulesClone # NB /a/b to make the relative path
 git clone $rootRepo $cloneRepo
 pathIndirect=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath")
 [[ $pathIndirect = $pathWithRelative ]]
+
+# Test submodule export-ignore interaction
+git -C $rootRepo/sub config user.email "foobar@example.com"
+git -C $rootRepo/sub config user.name "Foobar"
+
+echo "/exclude-from-root export-ignore" >> $rootRepo/.gitattributes
+echo nope > $rootRepo/exclude-from-root
+git -C $rootRepo add .gitattributes exclude-from-root
+git -C $rootRepo commit -m "Add export-ignore"
+
+echo "/exclude-from-sub export-ignore" >> $rootRepo/sub/.gitattributes
+echo nope > $rootRepo/sub/exclude-from-sub
+git -C $rootRepo/sub add .gitattributes exclude-from-sub
+git -C $rootRepo/sub commit -m "Add export-ignore (sub)"
+
+git -C $rootRepo add sub
+git -C $rootRepo commit -m "Update submodule"
+
+git -C $rootRepo status
+
+# exportIgnore can be used with submodules
+pathWithExportIgnore=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$rootRepo; submodules = true; exportIgnore = true; }).outPath")
+# find $pathWithExportIgnore
+# git -C $rootRepo archive --format=tar HEAD | tar -t
+# cp -a $rootRepo /tmp/rootRepo
+
+[[ -e $pathWithExportIgnore/sub/content ]]
+[[ ! -e $pathWithExportIgnore/exclude-from-root ]]
+[[ ! -e $pathWithExportIgnore/sub/exclude-from-sub ]]
+
+# exportIgnore can be explicitly disabled with submodules
+pathWithoutExportIgnore=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$rootRepo; submodules = true; exportIgnore = false; }).outPath")
+# find $pathWithoutExportIgnore
+
+[[ -e $pathWithoutExportIgnore/exclude-from-root ]]
+[[ -e $pathWithoutExportIgnore/sub/exclude-from-sub ]]
+
+# exportIgnore defaults to false when submodules = true
+pathWithSubmodules=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$rootRepo; submodules = true; }).outPath")
+
+[[ -e $pathWithoutExportIgnore/exclude-from-root ]]
+[[ -e $pathWithoutExportIgnore/sub/exclude-from-sub ]]

From 71d08af15bb2973dc2a1cb7fee18f94d779dfed7 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 5 Jan 2024 19:01:12 +0100
Subject: [PATCH 349/421] rl-next: Add *general* note about git fetcher reimpl

---
 doc/manual/rl-next/git-fetcher.md | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/manual/rl-next/git-fetcher.md

diff --git a/doc/manual/rl-next/git-fetcher.md b/doc/manual/rl-next/git-fetcher.md
new file mode 100644
index 000000000..54c0d216d
--- /dev/null
+++ b/doc/manual/rl-next/git-fetcher.md
@@ -0,0 +1,18 @@
+---
+synopsis: "Nix now uses `libgit2` for Git fetching"
+prs:
+  - 9240
+  - 9241
+  - 9258
+  - 9480
+issues:
+  - 5313
+---
+
+Nix has built-in support for fetching sources from Git, during evaluation and locking; outside the sandbox.
+The existing implementation based on the Git CLI had issues regarding reproducibility and performance.
+
+Most of the original `fetchGit` behavior has been implemented using the `libgit2` library, which gives the fetcher fine-grained control.
+
+Known issues:
+- The `export-subst` behavior has not been reimplemented. [Partial](https://github.com/NixOS/nix/pull/9391#issuecomment-1872503447) support for this Git feature is feasible, but it did not make the release window.

From 692e9197bc91f874ec30f839b1ae6d1beefa1eeb Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 5 Jan 2024 19:49:39 +0100
Subject: [PATCH 350/421] fetchTree: Disallow combination of submodules and
 exportIgnore for now

---
 src/libexpr/primops/fetchTree.cc       |  8 +++-----
 src/libfetchers/git.cc                 | 11 +++++++++++
 tests/functional/fetchGitSubmodules.sh | 26 ++++++++++++++++++--------
 3 files changed, 32 insertions(+), 13 deletions(-)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 7a4725334..4d22ca01e 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -1,3 +1,4 @@
+#include "libfetchers/attrs.hh"
 #include "primops.hh"
 #include "eval-inline.hh"
 #include "eval-settings.hh"
@@ -139,9 +140,7 @@ static void fetchTree(
                     state.symbols[attr.name], showType(*attr.value)));
         }
 
-        if (params.isFetchGit && !attrs.contains("exportIgnore")) {
-            // Default value; user attrs are assigned later.
-            // FIXME: exportIgnore := !submodules
+        if (params.isFetchGit && !attrs.contains("exportIgnore") && (!attrs.contains("submodules") || !*fetchers::maybeGetBoolAttr(attrs, "submodules"))) {
             attrs.emplace("exportIgnore", Explicit<bool>{true});
         }
 
@@ -162,8 +161,7 @@ static void fetchTree(
             fetchers::Attrs attrs;
             attrs.emplace("type", "git");
             attrs.emplace("url", fixGitURL(url));
-            if (!attrs.contains("exportIgnore")) {
-                // FIXME: exportIgnore := !submodules
+            if (!attrs.contains("exportIgnore") && (!attrs.contains("submodules") || !*fetchers::maybeGetBoolAttr(attrs, "submodules"))) {
                 attrs.emplace("exportIgnore", Explicit<bool>{true});
             }
             input = fetchers::Input::fromAttrs(std::move(attrs));
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 10c0aef97..6ecb7a4ea 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -1,3 +1,4 @@
+#include "error.hh"
 #include "fetchers.hh"
 #include "users.hh"
 #include "cache.hh"
@@ -739,6 +740,16 @@ struct GitInputScheme : InputScheme
 
         auto repoInfo = getRepoInfo(input);
 
+        if (getExportIgnoreAttr(input)
+            && getSubmodulesAttr(input)) {
+            /* In this situation, we don't have a git CLI behavior that we can copy.
+               `git archive` does not support submodules, so it is unclear whether
+               rules from the parent should affect the submodule or not.
+               When git may eventually implement this, we need Nix to match its
+               behavior. */
+            throw UnimplementedError("exportIgnore and submodules are not supported together yet");
+        }
+
         auto [accessor, final] =
             input.getRef() || input.getRev() || !repoInfo.isLocal
             ? getAccessorFromCommit(store, repoInfo, std::move(input))
diff --git a/tests/functional/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh
index 1b425820e..cd180815d 100644
--- a/tests/functional/fetchGitSubmodules.sh
+++ b/tests/functional/fetchGitSubmodules.sh
@@ -124,12 +124,16 @@ git -C $rootRepo/sub config user.email "foobar@example.com"
 git -C $rootRepo/sub config user.name "Foobar"
 
 echo "/exclude-from-root export-ignore" >> $rootRepo/.gitattributes
+# TBD possible semantics for submodules + exportIgnore
+# echo "/sub/exclude-deep export-ignore" >> $rootRepo/.gitattributes
 echo nope > $rootRepo/exclude-from-root
 git -C $rootRepo add .gitattributes exclude-from-root
 git -C $rootRepo commit -m "Add export-ignore"
 
 echo "/exclude-from-sub export-ignore" >> $rootRepo/sub/.gitattributes
 echo nope > $rootRepo/sub/exclude-from-sub
+# TBD possible semantics for submodules + exportIgnore
+# echo aye > $rootRepo/sub/exclude-from-root
 git -C $rootRepo/sub add .gitattributes exclude-from-sub
 git -C $rootRepo/sub commit -m "Add export-ignore (sub)"
 
@@ -138,15 +142,21 @@ git -C $rootRepo commit -m "Update submodule"
 
 git -C $rootRepo status
 
-# exportIgnore can be used with submodules
-pathWithExportIgnore=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$rootRepo; submodules = true; exportIgnore = true; }).outPath")
-# find $pathWithExportIgnore
-# git -C $rootRepo archive --format=tar HEAD | tar -t
-# cp -a $rootRepo /tmp/rootRepo
+# # TBD: not supported yet, because semantics are undecided and current implementation leaks rules from the root to submodules
+# # exportIgnore can be used with submodules
+# pathWithExportIgnore=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$rootRepo; submodules = true; exportIgnore = true; }).outPath")
+# # find $pathWithExportIgnore
+# # git -C $rootRepo archive --format=tar HEAD | tar -t
+# # cp -a $rootRepo /tmp/rootRepo
+
+# [[ -e $pathWithExportIgnore/sub/content ]]
+# [[ ! -e $pathWithExportIgnore/exclude-from-root ]]
+# [[ ! -e $pathWithExportIgnore/sub/exclude-from-sub ]]
+# TBD possible semantics for submodules + exportIgnore
+# # root .gitattribute has no power across submodule boundary
+# [[ -e $pathWithExportIgnore/sub/exclude-from-root ]]
+# [[ -e $pathWithExportIgnore/sub/exclude-deep ]]
 
-[[ -e $pathWithExportIgnore/sub/content ]]
-[[ ! -e $pathWithExportIgnore/exclude-from-root ]]
-[[ ! -e $pathWithExportIgnore/sub/exclude-from-sub ]]
 
 # exportIgnore can be explicitly disabled with submodules
 pathWithoutExportIgnore=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$rootRepo; submodules = true; exportIgnore = false; }).outPath")

From 469cf263c7d1b7991a9122b76b827f3d65a02301 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 12 Jan 2024 14:02:58 +0100
Subject: [PATCH 351/421] Format

---
 src/libfetchers/git-utils.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index cd65e0fda..b416c3b52 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -701,7 +701,8 @@ struct GitExportIgnoreInputAccessor : FilteringInputAccessor {
         }
     }
 
-    bool isExportIgnored(const CanonPath & path) {
+    bool isExportIgnored(const CanonPath & path)
+    {
         const char *exportIgnoreEntry = nullptr;
 
         // GIT_ATTR_CHECK_INDEX_ONLY:
@@ -721,7 +722,8 @@ struct GitExportIgnoreInputAccessor : FilteringInputAccessor {
         }
     }
 
-    bool isAllowed(const CanonPath & path) override {
+    bool isAllowed(const CanonPath & path) override
+    {
         return !isExportIgnored(path);
     }
 

From f68ad5acbb74c32d7ae6019bc17931940456603a Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 12 Jan 2024 16:05:36 +0100
Subject: [PATCH 352/421] fetchTree/git: Don't expose exportIgnore attr

---
 src/libexpr/primops/fetchTree.cc | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 4d22ca01e..7251cbbbe 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -40,10 +40,6 @@ void emitTreeAttrs(
         attrs.alloc("submodules").mkBool(
             fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
 
-    if (input.getType() == "git")
-        attrs.alloc("exportIgnore").mkBool(
-            fetchers::maybeGetBoolAttr(input.attrs, "exportIgnore").value_or(false));
-
     if (!forceDirty) {
 
         if (auto rev = input.getRev()) {

From 8c7e2ed77c3c14f8a7c82ab6ab7b20ebcfb943a0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 12 Jan 2024 16:21:07 +0100
Subject: [PATCH 353/421] Update release notes

---
 doc/manual/rl-next/nix-profile-names.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/manual/rl-next/nix-profile-names.md b/doc/manual/rl-next/nix-profile-names.md
index f5953bd72..b7ad4b5d7 100644
--- a/doc/manual/rl-next/nix-profile-names.md
+++ b/doc/manual/rl-next/nix-profile-names.md
@@ -3,4 +3,6 @@ synopsis: "`nix profile` now allows referring to elements by human-readable name
 prs: 8678
 ---
 
-[`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) now uses names to refer to installed packages when running [`list`](@docroot@/command-ref/new-cli/nix3-profile-list.md), [`remove`](@docroot@/command-ref/new-cli/nix3-profile-remove.md) or [`upgrade`](@docroot@/command-ref/new-cli/nix3-profile-upgrade.md) as opposed to indices. Indices are deprecated and will be removed in a future version.
+[`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) now uses names to refer to installed packages when running [`list`](@docroot@/command-ref/new-cli/nix3-profile-list.md), [`remove`](@docroot@/command-ref/new-cli/nix3-profile-remove.md) or [`upgrade`](@docroot@/command-ref/new-cli/nix3-profile-upgrade.md) as opposed to indices. Profile element names are generated when a package is installed and remain the same until the package is removed.
+
+**Warning**: The `manifest.nix` file used to record the contents of profiles has changed. Nix will automatically upgrade profiles to the new version when you modify the profile. After that, the profile can no longer be used by older versions of Nix.

From 72560f7bbef2ab3c02b8ca040fe084328bdd5fbe Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 12 Jan 2024 16:33:15 +0100
Subject: [PATCH 354/421] Add profile migration test

---
 tests/functional/nix-profile.sh | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh
index 003af5174..6f304bd9a 100644
--- a/tests/functional/nix-profile.sh
+++ b/tests/functional/nix-profile.sh
@@ -193,3 +193,12 @@ nix profile install $flake2Dir --priority 0
 clearProfiles
 nix profile install $(nix build $flake1Dir --no-link --print-out-paths)
 expect 1 nix profile install --impure --expr "(builtins.getFlake ''$flake2Dir'').packages.$system.default"
+
+# Test upgrading from profile version 2.
+clearProfiles
+mkdir -p $TEST_ROOT/import-profile
+outPath=$(nix build --no-link --print-out-paths $flake1Dir/flake.nix^out)
+printf '{ "version": 2, "elements": [ { "active": true, "attrPath": "legacyPackages.x86_64-linux.hello", "originalUrl": "flake:nixpkgs", "outputs": null, "priority": 5, "storePaths": [ "%s" ], "url": "github:NixOS/nixpkgs/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" } ] }' "$outPath" > $TEST_ROOT/import-profile/manifest.json
+nix build --profile $TEST_HOME/.nix-profile $(nix store add-path $TEST_ROOT/import-profile)
+nix profile list | grep -A4 'Name:.*hello' | grep "Store paths:.*$outPath"
+nix profile remove hello 2>&1 | grep 'removed 1 packages, kept 0 packages'

From d80c582b783e4c189432a2afd383be39cc09f17c Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 12 Jan 2024 17:16:59 +0100
Subject: [PATCH 355/421] libfetchers: Add CachingFilteringInputAccessor

Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
---
 src/libfetchers/filtering-input-accessor.cc |  9 +++++++++
 src/libfetchers/filtering-input-accessor.hh | 14 ++++++++++++++
 2 files changed, 23 insertions(+)

diff --git a/src/libfetchers/filtering-input-accessor.cc b/src/libfetchers/filtering-input-accessor.cc
index 5ae416fd3..581ce3c1d 100644
--- a/src/libfetchers/filtering-input-accessor.cc
+++ b/src/libfetchers/filtering-input-accessor.cc
@@ -80,4 +80,13 @@ ref<AllowListInputAccessor> AllowListInputAccessor::create(
     return make_ref<AllowListInputAccessorImpl>(next, std::move(allowedPaths), std::move(makeNotAllowedError));
 }
 
+bool CachingFilteringInputAccessor::isAllowed(const CanonPath & path)
+{
+    auto i = cache.find(path);
+    if (i != cache.end()) return i->second;
+    auto res = isAllowedUncached(path);
+    cache.emplace(path, res);
+    return res;
+}
+
 }
diff --git a/src/libfetchers/filtering-input-accessor.hh b/src/libfetchers/filtering-input-accessor.hh
index 2e2495c78..8a9b206ee 100644
--- a/src/libfetchers/filtering-input-accessor.hh
+++ b/src/libfetchers/filtering-input-accessor.hh
@@ -71,4 +71,18 @@ struct AllowListInputAccessor : public FilteringInputAccessor
     using FilteringInputAccessor::FilteringInputAccessor;
 };
 
+/**
+ * A wrapping `InputAccessor` mix-in where `isAllowed()` caches the result of virtual `isAllowedUncached()`.
+ */
+struct CachingFilteringInputAccessor : FilteringInputAccessor
+{
+    std::map<CanonPath, bool> cache;
+
+    using FilteringInputAccessor::FilteringInputAccessor;
+
+    bool isAllowed(const CanonPath & path) override;
+
+    virtual bool isAllowedUncached(const CanonPath & path) = 0;
+};
+
 }

From 274d887feee7e8bc3d4a7e6c5087fbe5aec4fd4e Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 12 Jan 2024 17:18:56 +0100
Subject: [PATCH 356/421] fetchTree/git: Cache export-ignore filter

---
 src/libfetchers/git-utils.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index b416c3b52..bfc7059fe 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -660,12 +660,12 @@ struct GitInputAccessor : InputAccessor
     }
 };
 
-struct GitExportIgnoreInputAccessor : FilteringInputAccessor {
+struct GitExportIgnoreInputAccessor : CachingFilteringInputAccessor {
     ref<GitRepoImpl> repo;
     std::optional<Hash> rev;
 
     GitExportIgnoreInputAccessor(ref<GitRepoImpl> repo, ref<InputAccessor> next, std::optional<Hash> rev)
-        : FilteringInputAccessor(next, [&](const CanonPath & path) {
+        : CachingFilteringInputAccessor(next, [&](const CanonPath & path) {
             return RestrictedPathError(fmt("'%s' does not exist because it was fetched with exportIgnore enabled", path));
         })
         , repo(repo)
@@ -722,7 +722,7 @@ struct GitExportIgnoreInputAccessor : FilteringInputAccessor {
         }
     }
 
-    bool isAllowed(const CanonPath & path) override
+    bool isAllowedUncached(const CanonPath & path) override
     {
         return !isExportIgnored(path);
     }

From 25c889baacd6a8b9b66ce4776ec729a40e39cf77 Mon Sep 17 00:00:00 2001
From: Mel Zuser <mel.zuser@arista.com>
Date: Thu, 11 Jan 2024 14:40:54 -0800
Subject: [PATCH 357/421] Fix performance of builtins.substring for empty
 substrings

When returning a 0-length substring, avoid calling coerceToString,
since it returns a string_view with the string's length, which is
expensive to compute for large strings.
---
 src/libexpr/primops.cc | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index ee07e5568..c08aea898 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -3712,9 +3712,6 @@ static RegisterPrimOp primop_toString({
 static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
     int start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring");
-    int len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring");
-    NixStringContext context;
-    auto s = state.coerceToString(pos, *args[2], context, "while evaluating the third argument (the string) passed to builtins.substring");
 
     if (start < 0)
         state.debugThrowLastTrace(EvalError({
@@ -3722,6 +3719,22 @@ static void prim_substring(EvalState & state, const PosIdx pos, Value * * args,
             .errPos = state.positions[pos]
         }));
 
+
+    int len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring");
+
+    // Special-case on empty substring to avoid O(n) strlen
+    // This allows for the use of empty substrings to efficently capture string context
+    if (len == 0) {
+        state.forceValue(*args[2], pos);
+        if (args[2]->type() == nString) {
+            v.mkString("", args[2]->context());
+            return;
+        }
+    }
+
+    NixStringContext context;
+    auto s = state.coerceToString(pos, *args[2], context, "while evaluating the third argument (the string) passed to builtins.substring");
+
     v.mkString((unsigned int) start >= s->size() ? "" : s->substr(start, len), context);
 }
 

From 6208ca72093a0e05c56561dab349423f9bff069b Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 1 Dec 2023 17:03:28 -0500
Subject: [PATCH 358/421] Separate `SystemError` from `SysError`

Most of this is a `catch SysError` -> `catch SystemError` sed. This
is a rather pure-churn change I would like to get out of the way. **The
intersting part is `src/libutil/error.hh`.**

On Unix, we will only throw the `SysError` concrete class, which has
the same constructors that `SystemError` used to have.

On Windows, we will throw `WinError` *and* `SysError`. `WinError`
(which will be created in a later PR), will use a `DWORD` instead of
`int` error value, and `GetLastError()`, which is the Windows equivalent
of the `errno` machinery. Windows will *also* use `SysError` because
Window's "libc" (MSVCRT) implements the POSIX interface, and we use it
too.

As the docs describe, while we *throw* one of the 3 choices above (2
concrete classes or the alias), we should always *catch* `SystemError`.
This ensures no matter how the implementation changes for Windows (e.g.
between `SysError` and `WinError`) the catching logic stays the same
and stays correct.

Co-Authored-By volth <volth@volth.com>
Co-Authored-By Eugene Butler <eugene@eugene4.com>
---
 src/libcmd/repl.cc                          |  2 +-
 src/libstore/build/local-derivation-goal.cc |  4 +-
 src/libstore/gc.cc                          |  2 +-
 src/libstore/globals.cc                     |  2 +-
 src/libstore/keys.cc                        |  2 +-
 src/libstore/local-store.cc                 |  2 +-
 src/libstore/optimise-store.cc              |  2 +-
 src/libstore/remote-fs-accessor.cc          |  4 +-
 src/libutil/args.cc                         |  2 +-
 src/libutil/cgroup.cc                       |  2 +-
 src/libutil/config.cc                       |  2 +-
 src/libutil/error.hh                        | 42 ++++++++++++++++++---
 src/libutil/file-descriptor.cc              |  2 +-
 src/libutil/logging.cc                      |  2 +-
 src/libutil/serialise.cc                    |  2 +-
 src/libutil/util.cc                         |  2 +-
 src/nix-build/nix-build.cc                  |  2 +-
 src/nix/config-check.cc                     |  2 +-
 tests/unit/libutil/logging.cc               |  6 +--
 19 files changed, 59 insertions(+), 27 deletions(-)

diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc
index 72e3559df..918b2e53a 100644
--- a/src/libcmd/repl.cc
+++ b/src/libcmd/repl.cc
@@ -254,7 +254,7 @@ void NixRepl::mainLoop()
     rl_readline_name = "nix-repl";
     try {
         createDirs(dirOf(historyFile));
-    } catch (SysError & e) {
+    } catch (SystemError & e) {
         logWarning(e.info());
     }
 #ifndef USE_READLINE
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index b01d9e237..f85301950 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -1495,7 +1495,7 @@ void LocalDerivationGoal::startDaemon()
                     daemon::processConnection(store, from, to,
                         NotTrusted, daemon::Recursive);
                     debug("terminated daemon connection");
-                } catch (SysError &) {
+                } catch (SystemError &) {
                     ignoreException();
                 }
             });
@@ -1707,7 +1707,7 @@ void LocalDerivationGoal::runChild()
         try {
             if (drv->isBuiltin() && drv->builder == "builtin:fetchurl")
                 netrcData = readFile(settings.netrcFile);
-        } catch (SysError &) { }
+        } catch (SystemError &) { }
 
 #if __linux__
         if (useChroot) {
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 2bd3a2edc..5cbce0748 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -413,7 +413,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
                     auto env_end = std::sregex_iterator{};
                     for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
                         unchecked[i->str()].emplace(envFile);
-                } catch (SysError & e) {
+                } catch (SystemError & e) {
                     if (errno == ENOENT || errno == EACCES || errno == ESRCH)
                         continue;
                     throw;
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 50584e06c..d22ae4ca0 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -118,7 +118,7 @@ void loadConfFile()
         try {
             std::string contents = readFile(path);
             globalConfig.applyConfig(contents, path);
-        } catch (SysError &) { }
+        } catch (SystemError &) { }
     };
 
     applyConfigFile(settings.nixConfDir + "/nix.conf");
diff --git a/src/libstore/keys.cc b/src/libstore/keys.cc
index 2cc50970f..70478e7ad 100644
--- a/src/libstore/keys.cc
+++ b/src/libstore/keys.cc
@@ -19,7 +19,7 @@ PublicKeys getDefaultPublicKeys()
         try {
             SecretKey secretKey(readFile(secretKeyFile));
             publicKeys.emplace(secretKey.name, secretKey.toPublicKey());
-        } catch (SysError & e) {
+        } catch (SystemError & e) {
             /* Ignore unreadable key files. That's normal in a
                multi-user installation. */
         }
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 0f3c37c8a..5a399c8be 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -276,7 +276,7 @@ LocalStore::LocalStore(const Params & params)
                 [[gnu::unused]] auto res2 = ftruncate(fd.get(), settings.reservedSize);
             }
         }
-    } catch (SysError & e) { /* don't care about errors */
+    } catch (SystemError & e) { /* don't care about errors */
     }
 
     /* Acquire the big fat lock in shared mode to make sure that no
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index a494e6ecc..78e4f6d86 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -242,7 +242,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     /* Atomically replace the old file with the new hard link. */
     try {
         renameFile(tempLink, path);
-    } catch (SysError & e) {
+    } catch (SystemError & e) {
         if (unlink(tempLink.c_str()) == -1)
             printError("unable to unlink '%1%'", tempLink);
         if (errno == EMLINK) {
diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc
index 03e57a565..b44edfe89 100644
--- a/src/libstore/remote-fs-accessor.cc
+++ b/src/libstore/remote-fs-accessor.cc
@@ -87,13 +87,13 @@ std::pair<ref<SourceAccessor>, CanonPath> RemoteFSAccessor::fetch(const CanonPat
             nars.emplace(storePath.hashPart(), narAccessor);
             return {narAccessor, restPath};
 
-        } catch (SysError &) { }
+        } catch (SystemError &) { }
 
         try {
             auto narAccessor = makeNarAccessor(nix::readFile(cacheFile));
             nars.emplace(storePath.hashPart(), narAccessor);
             return {narAccessor, restPath};
-        } catch (SysError &) { }
+        } catch (SystemError &) { }
     }
 
     StringSink sink;
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index e2668c673..5187e7396 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -304,7 +304,7 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang)
                 for (auto pos = savedArgs.begin(); pos != savedArgs.end();pos++)
                     cmdline.push_back(*pos);
             }
-        } catch (SysError &) { }
+        } catch (SystemError &) { }
     }
     for (auto pos = cmdline.begin(); pos != cmdline.end(); ) {
 
diff --git a/src/libutil/cgroup.cc b/src/libutil/cgroup.cc
index 4c2bf31ff..de83b5ad1 100644
--- a/src/libutil/cgroup.cc
+++ b/src/libutil/cgroup.cc
@@ -95,7 +95,7 @@ static CgroupStats destroyCgroup(const Path & cgroup, bool returnStats)
                     using namespace std::string_literals;
                     warn("killing stray builder process %d (%s)...",
                         pid, trim(replaceStrings(cmdline, "\0"s, " ")));
-                } catch (SysError &) {
+                } catch (SystemError &) {
                 }
             }
             // FIXME: pid wraparound
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index a3310f4ec..37f5b50c7 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -124,7 +124,7 @@ static void applyConfigInner(const std::string & contents, const std::string & p
                 try {
                     std::string includedContents = readFile(path);
                     applyConfigInner(includedContents, p, parsedContents);
-                } catch (SysError &) {
+                } catch (SystemError &) {
                     // TODO: Do we actually want to ignore this? Or is it better to fail?
                 }
             } else if (!ignoreMissing) {
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 234cbe1f6..764fac1ce 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -178,20 +178,50 @@ MakeError(Error, BaseError);
 MakeError(UsageError, Error);
 MakeError(UnimplementedError, Error);
 
-class SysError : public Error
+/**
+ * To use in catch-blocks.
+ */
+MakeError(SystemError, Error);
+
+/**
+ * POSIX system error, created using `errno`, `strerror` friends.
+ *
+ * Throw this, but prefer not to catch this, and catch `SystemError`
+ * instead. This allows implementations to freely switch between this
+ * and `WinError` without breaking catch blocks.
+ *
+ * However, it is permissible to catch this and rethrow so long as
+ * certain conditions are not met (e.g. to catch only if `errNo =
+ * EFooBar`). In that case, try to also catch the equivalent `WinError`
+ * code.
+ *
+ * @todo Rename this to `PosixError` or similar. At this point Windows
+ * support is too WIP to justify the code churn, but if it is finished
+ * then a better identifier becomes moe worth it.
+ */
+class SysError : public SystemError
 {
 public:
     int errNo;
 
+    /**
+     * Construct using the explicitly-provided error number. `strerror`
+     * will be used to try to add additional information to the message.
+     */
     template<typename... Args>
-    SysError(int errNo_, const Args & ... args)
-        : Error("")
+    SysError(int errNo, const Args & ... args)
+        : SystemError(""), errNo(errNo)
     {
-        errNo = errNo_;
         auto hf = hintfmt(args...);
         err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo));
     }
 
+    /**
+     * Construct using the ambient `errno`.
+     *
+     * Be sure to not perform another `errno`-modifying operation before
+     * calling this constructor!
+     */
     template<typename... Args>
     SysError(const Args & ... args)
         : SysError(errno, args ...)
@@ -199,7 +229,9 @@ public:
     }
 };
 
-/** Throw an exception for the purpose of checking that exception handling works; see 'initLibUtil()'.
+/**
+ * Throw an exception for the purpose of checking that exception
+ * handling works; see 'initLibUtil()'.
  */
 void throwExceptionSelfCheck();
 
diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc
index 38dd70c8e..692be3383 100644
--- a/src/libutil/file-descriptor.cc
+++ b/src/libutil/file-descriptor.cc
@@ -231,7 +231,7 @@ void closeMostFDs(const std::set<int> & exceptions)
             }
         }
         return;
-    } catch (SysError &) {
+    } catch (SystemError &) {
     }
 #endif
 
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 183aee2dc..d68ddacc0 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -116,7 +116,7 @@ void writeToStderr(std::string_view s)
 {
     try {
         writeFull(STDERR_FILENO, s, false);
-    } catch (SysError & e) {
+    } catch (SystemError & e) {
         /* Ignore failing writes to stderr.  We need to ignore write
            errors to ensure that cleanup code that logs to stderr runs
            to completion if the other side of stderr has been closed
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 76b378e18..316105603 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -53,7 +53,7 @@ void FdSink::writeUnbuffered(std::string_view data)
     written += data.size();
     try {
         writeFull(fd, data);
-    } catch (SysError & e) {
+    } catch (SystemError & e) {
         _good = false;
         throw;
     }
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 7b4b1d031..b23362b5c 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -20,7 +20,7 @@ void initLibUtil() {
     // When exception handling fails, the message tends to be printed by the
     // C++ runtime, followed by an abort.
     // For example on macOS we might see an error such as
-    // libc++abi: terminating with uncaught exception of type nix::SysError: error: C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded.
+    // libc++abi: terminating with uncaught exception of type nix::SystemError: error: C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded.
     bool caught = false;
     try {
         throwExceptionSelfCheck();
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index ee2addb72..1ad4b387c 100644
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -148,7 +148,7 @@ static void main_nix_build(int argc, char * * argv)
                             args.push_back(word);
                 }
             }
-        } catch (SysError &) { }
+        } catch (SystemError &) { }
     }
 
     struct MyArgs : LegacyArgs, MixEvalArgs
diff --git a/src/nix/config-check.cc b/src/nix/config-check.cc
index 410feca2f..8d4717e15 100644
--- a/src/nix/config-check.cc
+++ b/src/nix/config-check.cc
@@ -107,7 +107,7 @@ struct CmdConfigCheck : StoreCommand
                     if (profileDir.find("/profiles/") == std::string::npos)
                         dirs.insert(dir);
                 }
-            } catch (SysError &) {}
+            } catch (SystemError &) {}
         }
 
         if (!dirs.empty()) {
diff --git a/tests/unit/libutil/logging.cc b/tests/unit/libutil/logging.cc
index c6dfe63d3..8950a26d4 100644
--- a/tests/unit/libutil/logging.cc
+++ b/tests/unit/libutil/logging.cc
@@ -73,7 +73,7 @@ namespace nix {
 
     }
 
-    TEST(logEI, picksUpSysErrorExitCode) {
+    TEST(logEI, picksUpSystemErrorExitCode) {
 
         MakeError(TestError, Error);
         ErrorInfo::programName = std::optional("error-unit-test");
@@ -81,12 +81,12 @@ namespace nix {
         try {
             auto x = readFile(-1);
         }
-        catch (SysError &e) {
+        catch (SystemError &e) {
             testing::internal::CaptureStderr();
             logError(e.info());
             auto str = testing::internal::GetCapturedStderr();
 
-            ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nstatting file: \x1B[33;1mBad file descriptor\x1B[0m\n");
+            ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SystemError --- error-unit-test\x1B[0m\nstatting file: \x1B[33;1mBad file descriptor\x1B[0m\n");
         }
     }
 

From 1996105e91d8d2022869c4e66c0a0734e363052b Mon Sep 17 00:00:00 2001
From: Mel Zuser <mel.zuser@arista.com>
Date: Fri, 12 Jan 2024 08:57:08 -0800
Subject: [PATCH 359/421] added test for empty substring special case

---
 tests/functional/lang/eval-okay-substring-context.exp |  1 +
 tests/functional/lang/eval-okay-substring-context.nix | 11 +++++++++++
 2 files changed, 12 insertions(+)
 create mode 100644 tests/functional/lang/eval-okay-substring-context.exp
 create mode 100644 tests/functional/lang/eval-okay-substring-context.nix

diff --git a/tests/functional/lang/eval-okay-substring-context.exp b/tests/functional/lang/eval-okay-substring-context.exp
new file mode 100644
index 000000000..2fe7f71fa
--- /dev/null
+++ b/tests/functional/lang/eval-okay-substring-context.exp
@@ -0,0 +1 @@
+"okay"
diff --git a/tests/functional/lang/eval-okay-substring-context.nix b/tests/functional/lang/eval-okay-substring-context.nix
new file mode 100644
index 000000000..d0ef70d4e
--- /dev/null
+++ b/tests/functional/lang/eval-okay-substring-context.nix
@@ -0,0 +1,11 @@
+with builtins;
+
+let
+
+  s = "${builtins.derivation { name = "test"; builder = "/bin/sh"; system = "x86_64-linux"; }}";
+
+in
+
+if getContext s == getContext "${substring 0 0 s + unsafeDiscardStringContext s}"
+then "okay"
+else throw "empty substring should preserve context"

From b29be1ff57e6e358b2925012a13d7d4a0312560e Mon Sep 17 00:00:00 2001
From: Rebecca Turner <rbt@sent.as>
Date: Fri, 12 Jan 2024 10:01:55 -0800
Subject: [PATCH 360/421] Document unit tests in hacking.md

---
 doc/manual/src/contributing/hacking.md | 5 ++++-
 doc/manual/src/contributing/testing.md | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index 9a03ac9b6..0fa59e891 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -51,11 +51,14 @@ To install it in `$(pwd)/outputs` and test it:
 
 ```console
 [nix-shell]$ make install
-[nix-shell]$ make installcheck -j $NIX_BUILD_CORES
+[nix-shell]$ make installcheck check -j $NIX_BUILD_CORES
 [nix-shell]$ nix --version
 nix (Nix) 2.12
 ```
 
+For more information on running and filtering tests, see
+[`testing.md`](./testing.md).
+
 To build a release version of Nix for the current operating system and CPU architecture:
 
 ```console
diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md
index d8d162379..31c39c16c 100644
--- a/doc/manual/src/contributing/testing.md
+++ b/doc/manual/src/contributing/testing.md
@@ -77,7 +77,7 @@ there is no risk of any build-system wildcards for the library accidentally pick
 ### Running tests
 
 You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`.
-Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable.
+Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable, e.g. `GTEST_FILTER='ErrorTraceTest.*' make check`.
 
 ### Characterisation testing { #characaterisation-testing-unit }
 

From 2d96c7a51f04755dc22856be012bd73dec13ad13 Mon Sep 17 00:00:00 2001
From: Rebecca Turner <rbt@sent.as>
Date: Fri, 12 Jan 2024 11:27:31 -0800
Subject: [PATCH 361/421] Remove outdated reference to `y` in `nix-instantiate`
 man page

---
 doc/manual/src/command-ref/nix-instantiate.md | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md
index 483150aa8..89e106bb0 100644
--- a/doc/manual/src/command-ref/nix-instantiate.md
+++ b/doc/manual/src/command-ref/nix-instantiate.md
@@ -179,12 +179,7 @@ $ nix-instantiate --eval --xml --expr '{ x = {}; }'
     </attr>
   </attrs>
 </expr>
-```
 
-Note that `y` is left unevaluated (the XML representation doesn’t
-attempt to show non-normal forms).
-
-```console
 $ nix-instantiate --eval --xml --strict --expr '{ x = {}; }'
 <?xml version='1.0' encoding='utf-8'?>
 <expr>

From f73e50144f21adc9a6344bc4a5f8ded757d781fd Mon Sep 17 00:00:00 2001
From: Rebecca Turner <rbt@sent.as>
Date: Fri, 12 Jan 2024 11:31:49 -0800
Subject: [PATCH 362/421] Clarify ambiguity in `nix-instantiate` man page

---
 doc/manual/src/command-ref/nix-instantiate.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md
index 89e106bb0..479c9abcf 100644
--- a/doc/manual/src/command-ref/nix-instantiate.md
+++ b/doc/manual/src/command-ref/nix-instantiate.md
@@ -44,9 +44,10 @@ standard input.
 
     > **Warning**
     >
-    > This option produces ambiguous output which is not suitable for machine
-    > consumption. For example, these two Nix expressions print the same result
-    > despite having different types:
+    > This option produces output which can be parsed as a Nix expression which
+    > will produce a different result than the input expression when evaluated.
+    > For example, these two Nix expressions print the same result despite
+    > having different meaning:
     >
     > ```console
     > $ nix-instantiate --eval --expr '{ a = {}; }'

From 15f7bdaf276252f7b536c189b9b3eef73ad0e6e7 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 12 Jan 2024 22:55:37 +0100
Subject: [PATCH 363/421] CanonPath: Add rel_c_str()

Defensively because isRoot() is also defensive.
---
 src/libfetchers/git-utils.cc | 3 +--
 src/libutil/canon-path.hh    | 7 +++++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index bfc7059fe..6726407b5 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -674,8 +674,7 @@ struct GitExportIgnoreInputAccessor : CachingFilteringInputAccessor {
 
     bool gitAttrGet(const CanonPath & path, const char * attrName, const char * & valueOut)
     {
-        std::string pathStr {path.rel()};
-        const char * pathCStr = pathStr.c_str();
+        const char * pathCStr = path.rel_c_str();
 
         if (rev) {
             git_attr_options opts = GIT_ATTR_OPTIONS_INIT;
diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh
index 6aff4ec0d..997c8c731 100644
--- a/src/libutil/canon-path.hh
+++ b/src/libutil/canon-path.hh
@@ -88,6 +88,13 @@ public:
     std::string_view rel() const
     { return ((std::string_view) path).substr(1); }
 
+    const char * rel_c_str() const
+    {
+        auto cs = path.c_str();
+        assert(cs[0]); // for safety if invariant is broken
+        return &cs[1];
+    }
+
     struct Iterator
     {
         std::string_view remaining;

From dd7e7b0a30a0564741c40e70f33cbf1cd6130106 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 11 Jan 2024 11:26:03 -0500
Subject: [PATCH 364/421] Newer Nixpkgs, get `readline` on Windows
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Now `nix repl` an, in principle, work on that platform too.

Flake lock file updates:

• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/2c9c58e98243930f8cb70387934daa4bc8b00373' (2023-12-31)
  → 'github:NixOS/nixpkgs/86501af7f1d51915e6c335f90f2cab73d7704ef3' (2024-01-11)
---
 flake.lock  | 6 +++---
 package.nix | 1 -
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/flake.lock b/flake.lock
index ae98d789a..65e468e8b 100644
--- a/flake.lock
+++ b/flake.lock
@@ -34,11 +34,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1704018918,
-        "narHash": "sha256-erjg/HrpC9liEfm7oLqb8GXCqsxaFwIIPqCsknW5aFY=",
+        "lastModified": 1704982786,
+        "narHash": "sha256-w62+4HyaHafLWjvrC2Eto7bSnSJQtia8oqs3//mkpCU=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "2c9c58e98243930f8cb70387934daa4bc8b00373",
+        "rev": "86501af7f1d51915e6c335f90f2cab73d7704ef3",
         "type": "github"
       },
       "original": {
diff --git a/package.nix b/package.nix
index 37410dc2f..a632fd6ec 100644
--- a/package.nix
+++ b/package.nix
@@ -236,7 +236,6 @@ in {
     openssl
     sqlite
     xz
-  ] ++ lib.optionals (!stdenv.hostPlatform.isWindows) [
     ({ inherit readline editline; }.${readlineFlavor})
   ] ++ lib.optionals enableMarkdown [
     lowdown

From e739a5002dab199a6cf207e6e62b394fa77f8cb2 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 12 Jan 2024 19:46:48 -0500
Subject: [PATCH 365/421] Avoid Windows macros in the parser and lexer

`FLOAT`, `INT`, and `IN` are identifers taken by macros.

The name `IN_KW` is chosen to match `OR_KW`, which is presumably named
that way for the same reason of dodging macros.
---
 src/libexpr/lexer.l  |  6 +++---
 src/libexpr/parser.y | 12 ++++++------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index df2cbd06f..9addb3ae8 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -130,7 +130,7 @@ else        { return ELSE; }
 assert      { return ASSERT; }
 with        { return WITH; }
 let         { return LET; }
-in          { return IN; }
+in          { return IN_KW; }
 rec         { return REC; }
 inherit     { return INHERIT; }
 or          { return OR_KW; }
@@ -156,7 +156,7 @@ or          { return OR_KW; }
                       .errPos = data->state.positions[CUR_POS],
                   });
               }
-              return INT;
+              return INT_LIT;
             }
 {FLOAT}     { errno = 0;
               yylval->nf = strtod(yytext, 0);
@@ -165,7 +165,7 @@ or          { return OR_KW; }
                       .msg = hintfmt("invalid float '%1%'", yytext),
                       .errPos = data->state.positions[CUR_POS],
                   });
-              return FLOAT;
+              return FLOAT_LIT;
             }
 
 \$\{        { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index b331776f0..60bcfebf9 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -365,11 +365,11 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
 %type <id> attr
 %token <id> ID
 %token <str> STR IND_STR
-%token <n> INT
-%token <nf> FLOAT
+%token <n> INT_LIT
+%token <nf> FLOAT_LIT
 %token <path> PATH HPATH SPATH PATH_END
 %token <uri> URI
-%token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL OR_KW
+%token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW
 %token DOLLAR_CURLY /* == ${ */
 %token IND_STRING_OPEN IND_STRING_CLOSE
 %token ELLIPSIS
@@ -412,7 +412,7 @@ expr_function
     { $$ = new ExprAssert(CUR_POS, $2, $4); }
   | WITH expr ';' expr_function
     { $$ = new ExprWith(CUR_POS, $2, $4); }
-  | LET binds IN expr_function
+  | LET binds IN_KW expr_function
     { if (!$2->dynamicAttrs.empty())
         throw ParseError({
             .msg = hintfmt("dynamic attributes not allowed in let"),
@@ -482,8 +482,8 @@ expr_simple
       else
           $$ = new ExprVar(CUR_POS, data->symbols.create($1));
   }
-  | INT { $$ = new ExprInt($1); }
-  | FLOAT { $$ = new ExprFloat($1); }
+  | INT_LIT { $$ = new ExprInt($1); }
+  | FLOAT_LIT { $$ = new ExprFloat($1); }
   | '"' string_parts '"' { $$ = $2; }
   | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
       $$ = stripIndentation(CUR_POS, data->symbols, std::move(*$2));

From cbd5553d57ebf5d0532047165a2d81825424bd76 Mon Sep 17 00:00:00 2001
From: Qyriad <qyriad@qyriad.me>
Date: Sat, 13 Jan 2024 04:20:08 -0700
Subject: [PATCH 366/421] doc: provide context in glossary definitions (#9378)

---
 doc/manual/src/glossary.md | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md
index 07891175a..1fdb8b4dd 100644
--- a/doc/manual/src/glossary.md
+++ b/doc/manual/src/glossary.md
@@ -3,10 +3,10 @@
 - [derivation]{#gloss-derivation}
 
   A description of a build task. The result of a derivation is a
-  store object. Derivations are typically specified in Nix expressions
+  store object. Derivations declared in Nix expressions are specified
   using the [`derivation` primitive](./language/derivations.md). These are
   translated into low-level *store derivations* (implicitly by
-  `nix-env` and `nix-build`, or explicitly by `nix-instantiate`).
+  `nix-build`, or explicitly by `nix-instantiate`).
 
   [derivation]: #gloss-derivation
 
@@ -14,6 +14,7 @@
 
   A [derivation] represented as a `.drv` file in the [store].
   It has a [store path], like any [store object].
+  It is the [instantiated][instantiate] form of a derivation.
 
   Example: `/nix/store/g946hcz4c8mdvq2g8vxx42z51qb71rvp-git-2.38.1.drv`
 
@@ -23,9 +24,9 @@
 
 - [instantiate]{#gloss-instantiate}, instantiation
 
-  Translate a [derivation] into a [store derivation].
+  Save an evaluated [derivation] as a [store derivation] in the Nix [store].
 
-  See [`nix-instantiate`](./command-ref/nix-instantiate.md).
+  See [`nix-instantiate`](./command-ref/nix-instantiate.md), which produces a store derivation from a Nix expression that evaluates to a derivation.
 
   [instantiate]: #gloss-instantiate
 
@@ -66,7 +67,7 @@
 
   From the perspective of the location where Nix is invoked, the Nix store can be  referred to _local_ or _remote_.
   Only a [local store]{#gloss-local-store} exposes a location in the file system of the machine where Nix is invoked that allows access to store objects, typically `/nix/store`.
-  Local stores can be used for building [derivations](#derivation).
+  Local stores can be used for building [derivations](#gloss-derivation).
   See [Local Store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) for details.
 
   [store]: #gloss-store
@@ -168,9 +169,10 @@
 
   A high-level description of software packages and compositions
   thereof. Deploying software using Nix entails writing Nix
-  expressions for your packages. Nix expressions are translated to
-  derivations that are stored in the Nix store. These derivations can
-  then be built.
+  expressions for your packages. Nix expressions specify [derivations][derivation],
+  which are [instantiated][instantiate] into the Nix store as [store derivations][store derivation].
+  These derivations can then be [realised][realise] to produce
+  [outputs][output].
 
 - [reference]{#gloss-reference}
 
@@ -222,6 +224,9 @@
 
   The [store derivation] that produced an [output path].
 
+  The deriver for an output path can be queried with the `--deriver` option to
+  [`nix-store --query`](@docroot@/command-ref/nix-store/query.md).
+
 - [validity]{#gloss-validity}
 
   A store path is valid if all [store object]s in its [closure] can be read from the [store].

From e838ac98d4fc54774bcaaa30a72cd9d3da01befc Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sat, 13 Jan 2024 19:41:27 +0100
Subject: [PATCH 367/421] doc/glossary: Nix expression can be language
 expression

---
 doc/manual/src/glossary.md | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md
index 1fdb8b4dd..870b2c3c6 100644
--- a/doc/manual/src/glossary.md
+++ b/doc/manual/src/glossary.md
@@ -167,12 +167,13 @@
 
 - [Nix expression]{#gloss-nix-expression}
 
-  A high-level description of software packages and compositions
-  thereof. Deploying software using Nix entails writing Nix
-  expressions for your packages. Nix expressions specify [derivations][derivation],
-  which are [instantiated][instantiate] into the Nix store as [store derivations][store derivation].
-  These derivations can then be [realised][realise] to produce
-  [outputs][output].
+  1. Commonly, a high-level description of software packages and compositions
+    thereof. Deploying software using Nix entails writing Nix
+    expressions for your packages. Nix expressions specify [derivations][derivation],
+    which are [instantiated][instantiate] into the Nix store as [store derivations][store derivation].
+    These derivations can then be [realised][realise] to produce [outputs][output].
+
+  2. A syntactically valid use of the [Nix language]. For example, the contents of a `.nix` file form an expression.
 
 - [reference]{#gloss-reference}
 
@@ -287,3 +288,6 @@
   These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting.
 
   See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md).
+
+
+[Nix language]: ./language/index.md
\ No newline at end of file

From 7e5fa5c25ce585da5399038bc92980fddbb65d8b Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Sat, 13 Jan 2024 20:00:06 +0100
Subject: [PATCH 368/421] doc/glossary: Define package and package attribute
 set

A small step towards https://github.com/NixOS/nix/issues/6507

I believe this incomplete definition is one that can be agreed on.
It would be nice to define more, but considering that the issue
also proposes changes to the design, I believe we should hold off
on those.

As for the wording, we're dealing with some very general and vague
terms, that have to be treated with exactly the right amount of
vagueness to be effective.

I start out with a fairly abstract definition of package.
1. to establish a baseline so we know what we're talking about
2. so that we can go in and clarify that we have an extra, Nix-specific
   definition.

"Software" is notoriously ill-defined, so it makes a great qualifier
for package, which we don't really want to pin down either, because
that would just get us lost in discussion.
We can come back to this after we've done 6057 and a few years in a
desert cave.

Then comes the "package attribute set" definition.
I can already hear Valentin say "That's not even Nix's responsibility!"
and on some days I might even agree.
However, in our current reality, we have `nix-env`, `nix-build` and
`nix profile`, which query the `outputName` attribute - among others -
which just don't exist in the derivation.

For those who can't believe what they're reading:

    $ nix-build --expr 'with import ./. {}; bind // {outputName = "lib";}' --no-out-link
    this path will be fetched (1.16 MiB download, 3.72 MiB unpacked):
      /nix/store/rfk6klfx3z972gavxlw6iypnj6j806ma-bind-9.18.21-lib
    copying path '/nix/store/rfk6klfx3z972gavxlw6iypnj6j806ma-bind-9.18.21-lib' from 'https://cache.nixos.org'...
    /nix/store/rfk6klfx3z972gavxlw6iypnj6j806ma-bind-9.18.21-lib

and let me tell you that bind is not a library.

So anyway, that's also proof of why calling this a "derivation attrset" would be wrong, despite the type attribute.
---
 doc/manual/src/glossary.md | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md
index 1fdb8b4dd..5e3c0e024 100644
--- a/doc/manual/src/glossary.md
+++ b/doc/manual/src/glossary.md
@@ -271,6 +271,21 @@
 
   The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute.
 
+- [package]{#package}
+
+  1. A software package; typically a collection of programs, files and data.
+
+  2. A [package attribute set].
+
+- [package attribute set]{#package-attribute-set}
+
+  An [attribute set] containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as
+  - attributes that refer to the files of a [package], typically in the form of [derivation outputs](#output),
+  - attributes that declare something about how the package is supposed to be installed or used,
+  - other metadata or arbitrary attributes.
+
+  [package attribute set]: #package-attribute-set
+
 - [string interpolation]{#gloss-string-interpolation}
 
   Expanding expressions enclosed in `${ }` within a [string], [path], or [attribute name].

From bbcd9fcfc1216bd7d88fef7933766e616c7111d0 Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.e.helbling@outlook.com>
Date: Sat, 13 Jan 2024 11:27:04 -0800
Subject: [PATCH 369/421] Arbitrarily bring back some nix-daemon calls

This means that both `nix daemon` and `nix-daemon` will be (somewhat)
tested.
---
 tests/functional/build-remote-trustless-should-pass-1.sh | 2 +-
 tests/functional/nix-daemon-untrusting.sh                | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/functional/build-remote-trustless-should-pass-1.sh b/tests/functional/build-remote-trustless-should-pass-1.sh
index 736e280e4..516bdf092 100644
--- a/tests/functional/build-remote-trustless-should-pass-1.sh
+++ b/tests/functional/build-remote-trustless-should-pass-1.sh
@@ -2,7 +2,7 @@ source common.sh
 
 # Remote trusts us
 file=build-hook.nix
-prog='nix%20daemon'
+prog=nix-daemon
 proto=ssh-ng
 
 source build-remote-trustless.sh
diff --git a/tests/functional/nix-daemon-untrusting.sh b/tests/functional/nix-daemon-untrusting.sh
index c339b5833..bcdb70989 100755
--- a/tests/functional/nix-daemon-untrusting.sh
+++ b/tests/functional/nix-daemon-untrusting.sh
@@ -1,3 +1,3 @@
 #!/bin/sh
 
-exec nix daemon --force-untrusted "$@"
+exec nix-daemon --force-untrusted "$@"

From f61d951909a619b7a430d8d8aa739e310c7bf472 Mon Sep 17 00:00:00 2001
From: Las Safin <me@las.rs>
Date: Sat, 13 Jan 2024 19:27:20 +0000
Subject: [PATCH 370/421] Avoid unnecessary copy of goal log

The data was (accidentally?) copied into a std::string,
even though the string is immediately converted into a std::string_view.
The code has been changed to construct a std::string_view directly,
such that one copy less happens.
---
 src/libstore/build/worker.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 399ad47fd..974a9f510 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -449,7 +449,7 @@ void Worker::waitForInput()
                 } else {
                     printMsg(lvlVomit, "%1%: read %2% bytes",
                         goal->getName(), rd);
-                    std::string data((char *) buffer.data(), rd);
+                    std::string_view data((char *) buffer.data(), rd);
                     j->lastOutput = after;
                     goal->handleChildOutput(k, data);
                 }

From 03a6ca9b253c35b33e041dce595239968224e0d3 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 14 Jan 2024 15:25:24 -0500
Subject: [PATCH 371/421] `tests/functional/nix-profile.sh`: Add missing
 `--no-link`

Otherwise we get a stray `tests/functional/result`, which can cause
spurious failures later.

(I got a failure because the test temp dir effecting the store dir
changed. This caused a test later because Nix didn't want to remove the
old `result` because it wasn't pointing inside the new Nix store.)
---
 tests/functional/nix-profile.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh
index 6f304bd9a..35a62fbe2 100644
--- a/tests/functional/nix-profile.sh
+++ b/tests/functional/nix-profile.sh
@@ -199,6 +199,6 @@ clearProfiles
 mkdir -p $TEST_ROOT/import-profile
 outPath=$(nix build --no-link --print-out-paths $flake1Dir/flake.nix^out)
 printf '{ "version": 2, "elements": [ { "active": true, "attrPath": "legacyPackages.x86_64-linux.hello", "originalUrl": "flake:nixpkgs", "outputs": null, "priority": 5, "storePaths": [ "%s" ], "url": "github:NixOS/nixpkgs/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" } ] }' "$outPath" > $TEST_ROOT/import-profile/manifest.json
-nix build --profile $TEST_HOME/.nix-profile $(nix store add-path $TEST_ROOT/import-profile)
+nix build --profile $TEST_HOME/.nix-profile $(nix store add-path $TEST_ROOT/import-profile) --no-link
 nix profile list | grep -A4 'Name:.*hello' | grep "Store paths:.*$outPath"
 nix profile remove hello 2>&1 | grep 'removed 1 packages, kept 0 packages'

From dd42a4e3e9ec6d76d393e24f449f161b62579dc5 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 15 Jan 2024 08:04:46 -0500
Subject: [PATCH 372/421] flake.lock: Update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Flake lock file updates:

• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/86501af7f1d51915e6c335f90f2cab73d7704ef3' (2024-01-11)
  → 'github:NixOS/nixpkgs/a1982c92d8980a0114372973cbdfe0a307f1bdea' (2024-01-12)
---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 65e468e8b..f0efb4036 100644
--- a/flake.lock
+++ b/flake.lock
@@ -34,11 +34,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1704982786,
-        "narHash": "sha256-w62+4HyaHafLWjvrC2Eto7bSnSJQtia8oqs3//mkpCU=",
+        "lastModified": 1705033721,
+        "narHash": "sha256-K5eJHmL1/kev6WuqyqqbS1cdNnSidIZ3jeqJ7GbrYnQ=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "86501af7f1d51915e6c335f90f2cab73d7704ef3",
+        "rev": "a1982c92d8980a0114372973cbdfe0a307f1bdea",
         "type": "github"
       },
       "original": {

From 9b9ecdee3424056cb854bc8f1aa49fe330c08c83 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Thu, 11 Jan 2024 23:50:03 -0500
Subject: [PATCH 373/421] Simplify RapidCheck configure

No more `RAPIDCHECK_HEADERS`!
---
 Makefile.config.in              |  1 -
 configure.ac                    | 21 +--------------------
 doc/internal-api/doxygen.cfg.in |  2 +-
 package.nix                     |  2 +-
 4 files changed, 3 insertions(+), 23 deletions(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index 21a9f41ec..d5c382630 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -29,7 +29,6 @@ LOWDOWN_LIBS = @LOWDOWN_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@
 PACKAGE_VERSION = @PACKAGE_VERSION@
-RAPIDCHECK_HEADERS = @RAPIDCHECK_HEADERS@
 SHELL = @bash@
 SODIUM_LIBS = @SODIUM_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
diff --git a/configure.ac b/configure.ac
index 2594544ab..f46cff732 100644
--- a/configure.ac
+++ b/configure.ac
@@ -353,27 +353,8 @@ AS_IF([test "$ENABLE_UNIT_TESTS" == "yes"],[
 # Look for gtest.
 PKG_CHECK_MODULES([GTEST], [gtest_main])
 
-
 # Look for rapidcheck.
-AC_ARG_VAR([RAPIDCHECK_HEADERS], [include path of gtest headers shipped by RAPIDCHECK])
-# No pkg-config yet, https://github.com/emil-e/rapidcheck/issues/302
-AC_LANG_PUSH(C++)
-AC_SUBST(RAPIDCHECK_HEADERS)
-[CXXFLAGS="-I $RAPIDCHECK_HEADERS $CXXFLAGS"]
-[LIBS="-lrapidcheck -lgtest $LIBS"]
-AC_CHECK_HEADERS([rapidcheck/gtest.h], [], [], [#include <gtest/gtest.h>])
-dnl AC_CHECK_LIB doesn't work for C++ libs with mangled symbols
-AC_LINK_IFELSE([
-    AC_LANG_PROGRAM([[
-      #include <gtest/gtest.h>
-      #include <rapidcheck/gtest.h>
-    ]], [[
-      return RUN_ALL_TESTS();
-    ]])
-  ],
-  [],
-  [AC_MSG_ERROR([librapidcheck is not found.])])
-AC_LANG_POP(C++)
+PKG_CHECK_MODULES([RAPIDCHECK], [rapidcheck rapidcheck_gtest])
 
 ])
 
diff --git a/doc/internal-api/doxygen.cfg.in b/doc/internal-api/doxygen.cfg.in
index ad5af97e6..6c6c325bd 100644
--- a/doc/internal-api/doxygen.cfg.in
+++ b/doc/internal-api/doxygen.cfg.in
@@ -81,7 +81,7 @@ EXPAND_ONLY_PREDEF     = YES
 # RECURSIVE has no effect here.
 # This tag requires that the tag SEARCH_INCLUDES is set to YES.
 
-INCLUDE_PATH           = @RAPIDCHECK_HEADERS@
+INCLUDE_PATH           =
 
 # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
 # tag can be used to specify a list of macro names that should be expanded. The
diff --git a/package.nix b/package.nix
index a632fd6ec..a1188ba9c 100644
--- a/package.nix
+++ b/package.nix
@@ -309,7 +309,7 @@ in {
   ] ++ lib.optional (doBuild && stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux"))
        "LDFLAGS=-fuse-ld=gold"
     ++ lib.optional (doBuild && stdenv.hostPlatform.isStatic) "--enable-embedded-sandbox-shell"
-    ++ lib.optional buildUnitTests "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include";
+    ;
 
   enableParallelBuilding = true;
 

From beed00c04e136e8d685905e4b2b1116ecdf42f63 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 13 Jan 2024 13:08:38 -0500
Subject: [PATCH 374/421] `absPath`: just take a `std::string_view`

1. Slightly more efficient

2. Easier to call

Co-authored-by: Cole Helbling <cole.e.helbling@outlook.com>
---
 src/libutil/canon-path.cc  |  6 +++---
 src/libutil/file-system.cc | 14 +++++++++++---
 src/libutil/file-system.hh |  2 +-
 3 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc
index 1e465f1f6..0a0f96a05 100644
--- a/src/libutil/canon-path.cc
+++ b/src/libutil/canon-path.cc
@@ -6,11 +6,11 @@ namespace nix {
 CanonPath CanonPath::root = CanonPath("/");
 
 CanonPath::CanonPath(std::string_view raw)
-    : path(absPath((Path) raw, "/"))
+    : path(absPath(raw, "/"))
 { }
 
 CanonPath::CanonPath(std::string_view raw, const CanonPath & root)
-    : path(absPath((Path) raw, root.abs()))
+    : path(absPath(raw, root.abs()))
 { }
 
 CanonPath::CanonPath(const std::vector<std::string> & elems)
@@ -22,7 +22,7 @@ CanonPath::CanonPath(const std::vector<std::string> & elems)
 
 CanonPath CanonPath::fromCwd(std::string_view path)
 {
-    return CanonPath(unchecked_t(), absPath((Path) path));
+    return CanonPath(unchecked_t(), absPath(path));
 }
 
 std::optional<CanonPath> CanonPath::parent() const
diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc
index 4cac35ace..ab8d32275 100644
--- a/src/libutil/file-system.cc
+++ b/src/libutil/file-system.cc
@@ -21,9 +21,16 @@ namespace fs = std::filesystem;
 
 namespace nix {
 
-Path absPath(Path path, std::optional<PathView> dir, bool resolveSymlinks)
+Path absPath(PathView path, std::optional<PathView> dir, bool resolveSymlinks)
 {
+    std::string scratch;
+
     if (path[0] != '/') {
+        // In this case we need to call `canonPath` on a newly-created
+        // string. We set `scratch` to that string first, and then set
+        // `path` to `scratch`. This ensures the newly-created string
+        // lives long enough for the call to `canonPath`, and allows us
+        // to just accept a `std::string_view`.
         if (!dir) {
 #ifdef __GNU__
             /* GNU (aka. GNU/Hurd) doesn't have any limitation on path
@@ -35,12 +42,13 @@ Path absPath(Path path, std::optional<PathView> dir, bool resolveSymlinks)
             if (!getcwd(buf, sizeof(buf)))
 #endif
                 throw SysError("cannot get cwd");
-            path = concatStrings(buf, "/", path);
+            scratch = concatStrings(buf, "/", path);
 #ifdef __GNU__
             free(buf);
 #endif
         } else
-            path = concatStrings(*dir, "/", path);
+            scratch = concatStrings(*dir, "/", path);
+        path = scratch;
     }
     return canonPath(path, resolveSymlinks);
 }
diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh
index 4637507b3..464efc242 100644
--- a/src/libutil/file-system.hh
+++ b/src/libutil/file-system.hh
@@ -41,7 +41,7 @@ struct Source;
  * specified directory, or the current directory otherwise.  The path
  * is also canonicalised.
  */
-Path absPath(Path path,
+Path absPath(PathView path,
     std::optional<PathView> dir = {},
     bool resolveSymlinks = false);
 

From e0a76430861efbcfaf14c8b3691a091e6e72a8ed Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 15 Jan 2024 22:35:12 +0000
Subject: [PATCH 375/421] Bump cachix/install-nix-action from 24 to 25

Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 24 to 25.
- [Release notes](https://github.com/cachix/install-nix-action/releases)
- [Commits](https://github.com/cachix/install-nix-action/compare/v24...v25)

---
updated-dependencies:
- dependency-name: cachix/install-nix-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/ci.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index aa2551424..8d88de4b1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -20,7 +20,7 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v24
+    - uses: cachix/install-nix-action@v25
       with:
         # The sandbox would otherwise be disabled by default on Darwin
         extra_nix_config: "sandbox = true"
@@ -62,7 +62,7 @@ jobs:
       with:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v24
+    - uses: cachix/install-nix-action@v25
       with:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
     - uses: cachix/cachix-action@v13
@@ -84,7 +84,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v24
+    - uses: cachix/install-nix-action@v25
       with:
         install_url: '${{needs.installer.outputs.installerURL}}'
         install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
@@ -114,7 +114,7 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v24
+    - uses: cachix/install-nix-action@v25
       with:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV

From bf7754c0991c33146da9c339a71d661615afc93a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 15 Jan 2024 22:35:15 +0000
Subject: [PATCH 376/421] Bump cachix/cachix-action from 13 to 14

Bumps [cachix/cachix-action](https://github.com/cachix/cachix-action) from 13 to 14.
- [Release notes](https://github.com/cachix/cachix-action/releases)
- [Commits](https://github.com/cachix/cachix-action/compare/v13...v14)

---
updated-dependencies:
- dependency-name: cachix/cachix-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index aa2551424..878720acc 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -25,7 +25,7 @@ jobs:
         # The sandbox would otherwise be disabled by default on Darwin
         extra_nix_config: "sandbox = true"
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/cachix-action@v13
+    - uses: cachix/cachix-action@v14
       if: needs.check_secrets.outputs.cachix == 'true'
       with:
         name: '${{ env.CACHIX_NAME }}'
@@ -65,7 +65,7 @@ jobs:
     - uses: cachix/install-nix-action@v24
       with:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
-    - uses: cachix/cachix-action@v13
+    - uses: cachix/cachix-action@v14
       with:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
@@ -119,7 +119,7 @@ jobs:
         install_url: https://releases.nixos.org/nix/nix-2.13.3/install
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV
-    - uses: cachix/cachix-action@v13
+    - uses: cachix/cachix-action@v14
       if: needs.check_secrets.outputs.cachix == 'true'
       with:
         name: '${{ env.CACHIX_NAME }}'

From cbc319e9be3b29e3eb29a6e4cf08db1e0363d7bd Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Tue, 16 Jan 2024 12:18:02 +0100
Subject: [PATCH 377/421] tests/functional/lang: Test substring with negative
 length

---
 tests/functional/lang/eval-okay-substring.exp | 2 +-
 tests/functional/lang/eval-okay-substring.nix | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/functional/lang/eval-okay-substring.exp b/tests/functional/lang/eval-okay-substring.exp
index 6aace04b0..f48b4623a 100644
--- a/tests/functional/lang/eval-okay-substring.exp
+++ b/tests/functional/lang/eval-okay-substring.exp
@@ -1 +1 @@
-"ooxfoobarybarzobaabbc"
+"ooxfoobarybarzobaabbc_bad"
diff --git a/tests/functional/lang/eval-okay-substring.nix b/tests/functional/lang/eval-okay-substring.nix
index 424af00d9..54c97e162 100644
--- a/tests/functional/lang/eval-okay-substring.nix
+++ b/tests/functional/lang/eval-okay-substring.nix
@@ -19,3 +19,5 @@ substring 1 2 s
 + substring 3 1 s
 + "c"
 + substring 5 10 "perl"
++ "_"
++ substring 3 (-1) "tebbad"

From baea5f42c602c0233c3ed9c2d668409f86f901b9 Mon Sep 17 00:00:00 2001
From: Robert Hensing <roberth@users.noreply.github.com>
Date: Tue, 16 Jan 2024 14:50:53 +0100
Subject: [PATCH 378/421] doc/glossary: Simplify software package definition

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 doc/manual/src/glossary.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md
index 5e3c0e024..3c0570a44 100644
--- a/doc/manual/src/glossary.md
+++ b/doc/manual/src/glossary.md
@@ -273,7 +273,7 @@
 
 - [package]{#package}
 
-  1. A software package; typically a collection of programs, files and data.
+  1. A software package; a collection of files and other data.
 
   2. A [package attribute set].
 

From 0b1d93d2bae5fda9924f13246d7a667ce4392a4d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 16 Jan 2024 15:23:22 +0100
Subject: [PATCH 379/421] Sleep a bit between attempts to connect to the root
 server

---
 src/libstore/gc.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index f60011f95..cb820e2d5 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -154,6 +154,7 @@ void LocalStore::addTempRoot(const StorePath & path)
                 if (e.errNo == ECONNREFUSED || e.errNo == ENOENT) {
                     debug("GC socket connection refused: %s", e.msg());
                     fdRootsSocket->close();
+                    std::this_thread::sleep_for(std::chrono::milliseconds(100));
                     goto restart;
                 }
                 throw;

From d005bade7f3339cc68bee12ce13d863d51d54dc4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 16 Jan 2024 15:23:46 +0100
Subject: [PATCH 380/421] connect(): Propagate errno from the child process

This is necessary on macOS since addTempRoot() relies on errno.
---
 src/libutil/unix-domain-socket.cc | 39 +++++++++++++++++++++----------
 1 file changed, 27 insertions(+), 12 deletions(-)

diff --git a/src/libutil/unix-domain-socket.cc b/src/libutil/unix-domain-socket.cc
index 8949461d2..05bbb5ba3 100644
--- a/src/libutil/unix-domain-socket.cc
+++ b/src/libutil/unix-domain-socket.cc
@@ -1,6 +1,7 @@
 #include "file-system.hh"
 #include "processes.hh"
 #include "unix-domain-socket.hh"
+#include "util.hh"
 
 #include <sys/socket.h>
 #include <sys/un.h>
@@ -75,21 +76,35 @@ void connect(int fd, const std::string & path)
     addr.sun_family = AF_UNIX;
 
     if (path.size() + 1 >= sizeof(addr.sun_path)) {
+        Pipe pipe;
+        pipe.create();
         Pid pid = startProcess([&]() {
-            Path dir = dirOf(path);
-            if (chdir(dir.c_str()) == -1)
-                throw SysError("chdir to '%s' failed", dir);
-            std::string base(baseNameOf(path));
-            if (base.size() + 1 >= sizeof(addr.sun_path))
-                throw Error("socket path '%s' is too long", base);
-            memcpy(addr.sun_path, base.c_str(), base.size() + 1);
-            if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
-                throw SysError("cannot connect to socket at '%s'", path);
-            _exit(0);
+            try {
+                pipe.readSide.close();
+                Path dir = dirOf(path);
+                if (chdir(dir.c_str()) == -1)
+                    throw SysError("chdir to '%s' failed", dir);
+                std::string base(baseNameOf(path));
+                if (base.size() + 1 >= sizeof(addr.sun_path))
+                    throw Error("socket path '%s' is too long", base);
+                memcpy(addr.sun_path, base.c_str(), base.size() + 1);
+                if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+                    throw SysError("cannot connect to socket at '%s'", path);
+                writeFull(pipe.writeSide.get(), "0\n");
+            } catch (SysError & e) {
+                writeFull(pipe.writeSide.get(), fmt("%d\n", e.errNo));
+            } catch (...) {
+                writeFull(pipe.writeSide.get(), "-1\n");
+            }
         });
-        int status = pid.wait();
-        if (status != 0)
+        pipe.writeSide.close();
+        auto errNo = string2Int<int>(chomp(drainFD(pipe.readSide.get())));
+        if (!errNo || *errNo == -1)
             throw Error("cannot connect to socket at '%s'", path);
+        else if (*errNo > 0) {
+            errno = *errNo;
+            throw SysError("cannot connect to socket at '%s'", path);
+        }
     } else {
         memcpy(addr.sun_path, path.c_str(), path.size() + 1);
         if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)

From 65255edc9b0c2bbe8b0be50ac7b2671b50309ea8 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Tue, 16 Jan 2024 15:25:04 +0100
Subject: [PATCH 381/421] DerivationInfo -> PackageInfo

This does not yet resolve the coupling between packages and
derivations, but it makes the code more consistent with the
terminology, and it accentuates places where the coupling is
obvious, such as

         auto drvPath = packageInfo.queryDrvPath();
         if (!drvPath)
             throw Error("'%s' is not a derivation", what());

... which isn't wrong, and in my opinion, doesn't even look
wrong, because it just reflects the current logic.
However, I do like that we can now start to see in the code that
this coupling is perhaps a bit arbitrary.
After this rename, we can bring the DerivingPath concept into type
and start to lift this limitation.
---
 src/libcmd/installable-attr-path.cc    | 10 +--
 src/libcmd/installable-value.hh        |  2 +-
 src/libcmd/installables.hh             |  2 +-
 src/libcmd/repl.cc                     |  6 +-
 src/libexpr/get-drvs.cc                | 48 ++++++-------
 src/libexpr/get-drvs.hh                | 18 ++---
 src/nix-build/nix-build.cc             | 16 ++---
 src/nix-env/nix-env.cc                 | 94 +++++++++++++-------------
 src/nix-env/user-env.cc                |  8 +--
 src/nix-env/user-env.hh                |  4 +-
 src/nix-instantiate/nix-instantiate.cc |  2 +-
 src/nix/flake.cc                       |  6 +-
 src/nix/profile.cc                     |  6 +-
 13 files changed, 111 insertions(+), 111 deletions(-)

diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc
index 06e507872..3ec1c1614 100644
--- a/src/libcmd/installable-attr-path.cc
+++ b/src/libcmd/installable-attr-path.cc
@@ -58,22 +58,22 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths()
 
     Bindings & autoArgs = *cmd.getAutoArgs(*state);
 
-    DrvInfos drvInfos;
-    getDerivations(*state, *v, "", autoArgs, drvInfos, false);
+    PackageInfos packageInfos;
+    getDerivations(*state, *v, "", autoArgs, packageInfos, false);
 
     // Backward compatibility hack: group results by drvPath. This
     // helps keep .all output together.
     std::map<StorePath, OutputsSpec> byDrvPath;
 
-    for (auto & drvInfo : drvInfos) {
-        auto drvPath = drvInfo.queryDrvPath();
+    for (auto & packageInfo : packageInfos) {
+        auto drvPath = packageInfo.queryDrvPath();
         if (!drvPath)
             throw Error("'%s' is not a derivation", what());
 
         auto newOutputs = std::visit(overloaded {
             [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec {
                 std::set<std::string> outputsToInstall;
-                for (auto & output : drvInfo.queryOutputs(false, true))
+                for (auto & output : packageInfo.queryOutputs(false, true))
                     outputsToInstall.insert(output.first);
                 return OutputsSpec::Names { std::move(outputsToInstall) };
             },
diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh
index 3138ce8ec..f300d392b 100644
--- a/src/libcmd/installable-value.hh
+++ b/src/libcmd/installable-value.hh
@@ -6,7 +6,7 @@
 
 namespace nix {
 
-struct DrvInfo;
+struct PackageInfo;
 struct SourceExprCommand;
 
 namespace eval_cache { class EvalCache; class AttrCursor; }
diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh
index 95e8841ca..bf5759230 100644
--- a/src/libcmd/installables.hh
+++ b/src/libcmd/installables.hh
@@ -12,7 +12,7 @@
 
 namespace nix {
 
-struct DrvInfo;
+struct PackageInfo;
 
 enum class Realise {
     /**
diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc
index 918b2e53a..d7d8f9819 100644
--- a/src/libcmd/repl.cc
+++ b/src/libcmd/repl.cc
@@ -450,10 +450,10 @@ static bool isVarName(std::string_view s)
 
 
 StorePath NixRepl::getDerivationPath(Value & v) {
-    auto drvInfo = getDerivation(*state, v, false);
-    if (!drvInfo)
+    auto packageInfo = getDerivation(*state, v, false);
+    if (!packageInfo)
         throw Error("expression does not evaluate to a derivation, so I can't build it");
-    auto drvPath = drvInfo->queryDrvPath();
+    auto drvPath = packageInfo->queryDrvPath();
     if (!drvPath)
         throw Error("expression did not evaluate to a valid derivation (no 'drvPath' attribute)");
     if (!state->store->isValidPath(*drvPath))
diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index a6441871c..51449ccb3 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -11,13 +11,13 @@
 namespace nix {
 
 
-DrvInfo::DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs)
+PackageInfo::PackageInfo(EvalState & state, std::string attrPath, Bindings * attrs)
     : state(&state), attrs(attrs), attrPath(std::move(attrPath))
 {
 }
 
 
-DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs)
+PackageInfo::PackageInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs)
     : state(&state), attrs(nullptr), attrPath("")
 {
     auto [drvPath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs);
@@ -45,7 +45,7 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
 }
 
 
-std::string DrvInfo::queryName() const
+std::string PackageInfo::queryName() const
 {
     if (name == "" && attrs) {
         auto i = attrs->find(state->sName);
@@ -56,7 +56,7 @@ std::string DrvInfo::queryName() const
 }
 
 
-std::string DrvInfo::querySystem() const
+std::string PackageInfo::querySystem() const
 {
     if (system == "" && attrs) {
         auto i = attrs->find(state->sSystem);
@@ -66,7 +66,7 @@ std::string DrvInfo::querySystem() const
 }
 
 
-std::optional<StorePath> DrvInfo::queryDrvPath() const
+std::optional<StorePath> PackageInfo::queryDrvPath() const
 {
     if (!drvPath && attrs) {
         Bindings::iterator i = attrs->find(state->sDrvPath);
@@ -80,7 +80,7 @@ std::optional<StorePath> DrvInfo::queryDrvPath() const
 }
 
 
-StorePath DrvInfo::requireDrvPath() const
+StorePath PackageInfo::requireDrvPath() const
 {
     if (auto drvPath = queryDrvPath())
         return *drvPath;
@@ -88,7 +88,7 @@ StorePath DrvInfo::requireDrvPath() const
 }
 
 
-StorePath DrvInfo::queryOutPath() const
+StorePath PackageInfo::queryOutPath() const
 {
     if (!outPath && attrs) {
         Bindings::iterator i = attrs->find(state->sOutPath);
@@ -102,7 +102,7 @@ StorePath DrvInfo::queryOutPath() const
 }
 
 
-DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall)
+PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall)
 {
     if (outputs.empty()) {
         /* Get the ‘outputs’ list. */
@@ -164,7 +164,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall
 }
 
 
-std::string DrvInfo::queryOutputName() const
+std::string PackageInfo::queryOutputName() const
 {
     if (outputName == "" && attrs) {
         Bindings::iterator i = attrs->find(state->sOutputName);
@@ -174,7 +174,7 @@ std::string DrvInfo::queryOutputName() const
 }
 
 
-Bindings * DrvInfo::getMeta()
+Bindings * PackageInfo::getMeta()
 {
     if (meta) return meta;
     if (!attrs) return 0;
@@ -186,7 +186,7 @@ Bindings * DrvInfo::getMeta()
 }
 
 
-StringSet DrvInfo::queryMetaNames()
+StringSet PackageInfo::queryMetaNames()
 {
     StringSet res;
     if (!getMeta()) return res;
@@ -196,7 +196,7 @@ StringSet DrvInfo::queryMetaNames()
 }
 
 
-bool DrvInfo::checkMeta(Value & v)
+bool PackageInfo::checkMeta(Value & v)
 {
     state->forceValue(v, v.determinePos(noPos));
     if (v.type() == nList) {
@@ -216,7 +216,7 @@ bool DrvInfo::checkMeta(Value & v)
 }
 
 
-Value * DrvInfo::queryMeta(const std::string & name)
+Value * PackageInfo::queryMeta(const std::string & name)
 {
     if (!getMeta()) return 0;
     Bindings::iterator a = meta->find(state->symbols.create(name));
@@ -225,7 +225,7 @@ Value * DrvInfo::queryMeta(const std::string & name)
 }
 
 
-std::string DrvInfo::queryMetaString(const std::string & name)
+std::string PackageInfo::queryMetaString(const std::string & name)
 {
     Value * v = queryMeta(name);
     if (!v || v->type() != nString) return "";
@@ -233,7 +233,7 @@ std::string DrvInfo::queryMetaString(const std::string & name)
 }
 
 
-NixInt DrvInfo::queryMetaInt(const std::string & name, NixInt def)
+NixInt PackageInfo::queryMetaInt(const std::string & name, NixInt def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
@@ -247,7 +247,7 @@ NixInt DrvInfo::queryMetaInt(const std::string & name, NixInt def)
     return def;
 }
 
-NixFloat DrvInfo::queryMetaFloat(const std::string & name, NixFloat def)
+NixFloat PackageInfo::queryMetaFloat(const std::string & name, NixFloat def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
@@ -262,7 +262,7 @@ NixFloat DrvInfo::queryMetaFloat(const std::string & name, NixFloat def)
 }
 
 
-bool DrvInfo::queryMetaBool(const std::string & name, bool def)
+bool PackageInfo::queryMetaBool(const std::string & name, bool def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
@@ -277,7 +277,7 @@ bool DrvInfo::queryMetaBool(const std::string & name, bool def)
 }
 
 
-void DrvInfo::setMeta(const std::string & name, Value * v)
+void PackageInfo::setMeta(const std::string & name, Value * v)
 {
     getMeta();
     auto attrs = state->buildBindings(1 + (meta ? meta->size() : 0));
@@ -300,7 +300,7 @@ typedef std::set<Bindings *> Done;
    The result boolean indicates whether it makes sense
    for the caller to recursively search for derivations in `v'. */
 static bool getDerivation(EvalState & state, Value & v,
-    const std::string & attrPath, DrvInfos & drvs, Done & done,
+    const std::string & attrPath, PackageInfos & drvs, Done & done,
     bool ignoreAssertionFailures)
 {
     try {
@@ -311,7 +311,7 @@ static bool getDerivation(EvalState & state, Value & v,
            derivation {...}; y = x;}'. */
         if (!done.insert(v.attrs).second) return false;
 
-        DrvInfo drv(state, attrPath, v.attrs);
+        PackageInfo drv(state, attrPath, v.attrs);
 
         drv.queryName();
 
@@ -326,11 +326,11 @@ static bool getDerivation(EvalState & state, Value & v,
 }
 
 
-std::optional<DrvInfo> getDerivation(EvalState & state, Value & v,
+std::optional<PackageInfo> getDerivation(EvalState & state, Value & v,
     bool ignoreAssertionFailures)
 {
     Done done;
-    DrvInfos drvs;
+    PackageInfos drvs;
     getDerivation(state, v, "", drvs, done, ignoreAssertionFailures);
     if (drvs.size() != 1) return {};
     return std::move(drvs.front());
@@ -348,7 +348,7 @@ static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*");
 
 static void getDerivations(EvalState & state, Value & vIn,
     const std::string & pathPrefix, Bindings & autoArgs,
-    DrvInfos & drvs, Done & done,
+    PackageInfos & drvs, Done & done,
     bool ignoreAssertionFailures)
 {
     Value v;
@@ -401,7 +401,7 @@ static void getDerivations(EvalState & state, Value & vIn,
 
 
 void getDerivations(EvalState & state, Value & v, const std::string & pathPrefix,
-    Bindings & autoArgs, DrvInfos & drvs, bool ignoreAssertionFailures)
+    Bindings & autoArgs, PackageInfos & drvs, bool ignoreAssertionFailures)
 {
     Done done;
     getDerivations(state, v, pathPrefix, autoArgs, drvs, done, ignoreAssertionFailures);
diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh
index 584d64ac1..b886581b6 100644
--- a/src/libexpr/get-drvs.hh
+++ b/src/libexpr/get-drvs.hh
@@ -11,7 +11,7 @@
 namespace nix {
 
 
-struct DrvInfo
+struct PackageInfo
 {
 public:
     typedef std::map<std::string, std::optional<StorePath>> Outputs;
@@ -43,9 +43,9 @@ public:
      */
     std::string attrPath;
 
-    DrvInfo(EvalState & state) : state(&state) { };
-    DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs);
-    DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs);
+    PackageInfo(EvalState & state) : state(&state) { };
+    PackageInfo(EvalState & state, std::string attrPath, Bindings * attrs);
+    PackageInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs);
 
     std::string queryName() const;
     std::string querySystem() const;
@@ -82,21 +82,21 @@ public:
 
 
 #if HAVE_BOEHMGC
-typedef std::list<DrvInfo, traceable_allocator<DrvInfo>> DrvInfos;
+typedef std::list<PackageInfo, traceable_allocator<PackageInfo>> PackageInfos;
 #else
-typedef std::list<DrvInfo> DrvInfos;
+typedef std::list<PackageInfo> PackageInfos;
 #endif
 
 
 /**
- * If value `v` denotes a derivation, return a DrvInfo object
+ * If value `v` denotes a derivation, return a PackageInfo object
  * describing it. Otherwise return nothing.
  */
-std::optional<DrvInfo> getDerivation(EvalState & state,
+std::optional<PackageInfo> getDerivation(EvalState & state,
     Value & v, bool ignoreAssertionFailures);
 
 void getDerivations(EvalState & state, Value & v, const std::string & pathPrefix,
-    Bindings & autoArgs, DrvInfos & drvs,
+    Bindings & autoArgs, PackageInfos & drvs,
     bool ignoreAssertionFailures);
 
 
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 1ad4b387c..549adfbf7 100644
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -289,7 +289,7 @@ static void main_nix_build(int argc, char * * argv)
     if (runEnv)
         setenv("IN_NIX_SHELL", pure ? "pure" : "impure", 1);
 
-    DrvInfos drvs;
+    PackageInfos drvs;
 
     /* Parse the expressions. */
     std::vector<Expr *> exprs;
@@ -307,7 +307,7 @@ static void main_nix_build(int argc, char * * argv)
                 } catch (Error & e) {};
                 auto [path, outputNames] = parsePathWithOutputs(absolute);
                 if (evalStore->isStorePath(path) && hasSuffix(path, ".drv"))
-                    drvs.push_back(DrvInfo(*state, evalStore, absolute));
+                    drvs.push_back(PackageInfo(*state, evalStore, absolute));
                 else
                     /* If we're in a #! script, interpret filenames
                        relative to the script. */
@@ -383,8 +383,8 @@ static void main_nix_build(int argc, char * * argv)
         if (drvs.size() != 1)
             throw UsageError("nix-shell requires a single derivation");
 
-        auto & drvInfo = drvs.front();
-        auto drv = evalStore->derivationFromPath(drvInfo.requireDrvPath());
+        auto & packageInfo = drvs.front();
+        auto drv = evalStore->derivationFromPath(packageInfo.requireDrvPath());
 
         std::vector<DerivedPath> pathsToBuild;
         RealisedPath::Set pathsToCopy;
@@ -527,7 +527,7 @@ static void main_nix_build(int argc, char * * argv)
             for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
                 accumInputClosure(inputDrv, inputNode);
 
-            ParsedDerivation parsedDrv(drvInfo.requireDrvPath(), drv);
+            ParsedDerivation parsedDrv(packageInfo.requireDrvPath(), drv);
 
             if (auto structAttrs = parsedDrv.prepareStructuredAttrs(*store, inputs)) {
                 auto json = structAttrs.value();
@@ -620,10 +620,10 @@ static void main_nix_build(int argc, char * * argv)
 
         std::map<StorePath, std::pair<size_t, StringSet>> drvMap;
 
-        for (auto & drvInfo : drvs) {
-            auto drvPath = drvInfo.requireDrvPath();
+        for (auto & packageInfo : drvs) {
+            auto drvPath = packageInfo.requireDrvPath();
 
-            auto outputName = drvInfo.queryOutputName();
+            auto outputName = packageInfo.queryOutputName();
             if (outputName == "")
                 throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath));
 
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index e2bbd9775..d5b46c57a 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -184,7 +184,7 @@ static void loadSourceExpr(EvalState & state, const SourcePath & path, Value & v
 
 static void loadDerivations(EvalState & state, const SourcePath & nixExprPath,
     std::string systemFilter, Bindings & autoArgs,
-    const std::string & pathPrefix, DrvInfos & elems)
+    const std::string & pathPrefix, PackageInfos & elems)
 {
     Value vRoot;
     loadSourceExpr(state, nixExprPath, vRoot);
@@ -195,7 +195,7 @@ static void loadDerivations(EvalState & state, const SourcePath & nixExprPath,
 
     /* Filter out all derivations not applicable to the current
        system. */
-    for (DrvInfos::iterator i = elems.begin(), j; i != elems.end(); i = j) {
+    for (PackageInfos::iterator i = elems.begin(), j; i != elems.end(); i = j) {
         j = i; j++;
         if (systemFilter != "*" && i->querySystem() != systemFilter)
             elems.erase(i);
@@ -203,13 +203,13 @@ static void loadDerivations(EvalState & state, const SourcePath & nixExprPath,
 }
 
 
-static long getPriority(EvalState & state, DrvInfo & drv)
+static long getPriority(EvalState & state, PackageInfo & drv)
 {
     return drv.queryMetaInt("priority", 0);
 }
 
 
-static long comparePriorities(EvalState & state, DrvInfo & drv1, DrvInfo & drv2)
+static long comparePriorities(EvalState & state, PackageInfo & drv1, PackageInfo & drv2)
 {
     return getPriority(state, drv2) - getPriority(state, drv1);
 }
@@ -217,7 +217,7 @@ static long comparePriorities(EvalState & state, DrvInfo & drv1, DrvInfo & drv2)
 
 // FIXME: this function is rather slow since it checks a single path
 // at a time.
-static bool isPrebuilt(EvalState & state, DrvInfo & elem)
+static bool isPrebuilt(EvalState & state, PackageInfo & elem)
 {
     auto path = elem.queryOutPath();
     if (state.store->isValidPath(path)) return true;
@@ -236,11 +236,11 @@ static void checkSelectorUse(DrvNames & selectors)
 
 namespace {
 
-std::set<std::string> searchByPrefix(const DrvInfos & allElems, std::string_view prefix) {
+std::set<std::string> searchByPrefix(const PackageInfos & allElems, std::string_view prefix) {
     constexpr std::size_t maxResults = 3;
     std::set<std::string> result;
-    for (const auto & drvInfo : allElems) {
-        const auto drvName = DrvName { drvInfo.queryName() };
+    for (const auto & packageInfo : allElems) {
+        const auto drvName = DrvName { packageInfo.queryName() };
         if (hasPrefix(drvName.name, prefix)) {
             result.emplace(drvName.name);
 
@@ -254,11 +254,11 @@ std::set<std::string> searchByPrefix(const DrvInfos & allElems, std::string_view
 
 struct Match
 {
-    DrvInfo drvInfo;
+    PackageInfo packageInfo;
     std::size_t index;
 
-    Match(DrvInfo drvInfo_, std::size_t index_)
-    : drvInfo{std::move(drvInfo_)}
+    Match(PackageInfo packageInfo_, std::size_t index_)
+    : packageInfo{std::move(packageInfo_)}
     , index{index_}
     {}
 };
@@ -276,7 +276,7 @@ std::vector<Match> pickNewestOnly(EvalState & state, std::vector<Match> matches)
     StringSet multiple;
 
     for (auto & match : matches) {
-        auto & oneDrv = match.drvInfo;
+        auto & oneDrv = match.packageInfo;
 
         const auto drvName = DrvName { oneDrv.queryName() };
         long comparison = 1;
@@ -284,7 +284,7 @@ std::vector<Match> pickNewestOnly(EvalState & state, std::vector<Match> matches)
         const auto itOther = newest.find(drvName.name);
 
         if (itOther != newest.end()) {
-            auto & newestDrv = itOther->second.drvInfo;
+            auto & newestDrv = itOther->second.packageInfo;
 
             comparison =
                 oneDrv.querySystem() == newestDrv.querySystem() ? 0 :
@@ -319,23 +319,23 @@ std::vector<Match> pickNewestOnly(EvalState & state, std::vector<Match> matches)
 
 } // end namespace
 
-static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
+static PackageInfos filterBySelector(EvalState & state, const PackageInfos & allElems,
     const Strings & args, bool newestOnly)
 {
     DrvNames selectors = drvNamesFromArgs(args);
     if (selectors.empty())
         selectors.emplace_back("*");
 
-    DrvInfos elems;
+    PackageInfos elems;
     std::set<std::size_t> done;
 
     for (auto & selector : selectors) {
         std::vector<Match> matches;
-        for (const auto & [index, drvInfo] : enumerate(allElems)) {
-            const auto drvName = DrvName { drvInfo.queryName() };
+        for (const auto & [index, packageInfo] : enumerate(allElems)) {
+            const auto drvName = DrvName { packageInfo.queryName() };
             if (selector.matches(drvName)) {
                 ++selector.hits;
-                matches.emplace_back(drvInfo, index);
+                matches.emplace_back(packageInfo, index);
             }
         }
 
@@ -347,7 +347,7 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
            haven't inserted before. */
         for (auto & match : matches)
             if (done.insert(match.index).second)
-                elems.push_back(match.drvInfo);
+                elems.push_back(match.packageInfo);
 
         if (selector.hits == 0 && selector.fullName != "*") {
             const auto prefixHits = searchByPrefix(allElems, selector.name);
@@ -376,7 +376,7 @@ static bool isPath(std::string_view s)
 
 static void queryInstSources(EvalState & state,
     InstallSourceInfo & instSource, const Strings & args,
-    DrvInfos & elems, bool newestOnly)
+    PackageInfos & elems, bool newestOnly)
 {
     InstallSourceType type = instSource.type;
     if (type == srcUnknown && args.size() > 0 && isPath(args.front()))
@@ -392,7 +392,7 @@ static void queryInstSources(EvalState & state,
 
             /* Load the derivations from the (default or specified)
                Nix expression. */
-            DrvInfos allElems;
+            PackageInfos allElems;
             loadDerivations(state, *instSource.nixExprPath,
                 instSource.systemFilter, *instSource.autoArgs, "", allElems);
 
@@ -433,7 +433,7 @@ static void queryInstSources(EvalState & state,
 
                 std::string name(path.name());
 
-                DrvInfo elem(state, "", nullptr);
+                PackageInfo elem(state, "", nullptr);
                 elem.setName(name);
 
                 if (path.isDerivation()) {
@@ -476,7 +476,7 @@ static void queryInstSources(EvalState & state,
 }
 
 
-static void printMissing(EvalState & state, DrvInfos & elems)
+static void printMissing(EvalState & state, PackageInfos & elems)
 {
     std::vector<DerivedPath> targets;
     for (auto & i : elems)
@@ -494,7 +494,7 @@ static void printMissing(EvalState & state, DrvInfos & elems)
 }
 
 
-static bool keep(DrvInfo & drv)
+static bool keep(PackageInfo & drv)
 {
     return drv.queryMetaBool("keep", false);
 }
@@ -506,7 +506,7 @@ static void installDerivations(Globals & globals,
     debug("installing derivations");
 
     /* Get the set of user environment elements to be installed. */
-    DrvInfos newElems, newElemsTmp;
+    PackageInfos newElems, newElemsTmp;
     queryInstSources(*globals.state, globals.instSource, args, newElemsTmp, true);
 
     /* If --prebuilt-only is given, filter out source-only packages. */
@@ -529,12 +529,12 @@ static void installDerivations(Globals & globals,
     while (true) {
         auto lockToken = optimisticLockProfile(profile);
 
-        DrvInfos allElems(newElems);
+        PackageInfos allElems(newElems);
 
         /* Add in the already installed derivations, unless they have
            the same name as a to-be-installed element. */
         if (!globals.removeAll) {
-            DrvInfos installedElems = queryInstalled(*globals.state, profile);
+            PackageInfos installedElems = queryInstalled(*globals.state, profile);
 
             for (auto & i : installedElems) {
                 DrvName drvName(i.queryName());
@@ -592,14 +592,14 @@ static void upgradeDerivations(Globals & globals,
     while (true) {
         auto lockToken = optimisticLockProfile(globals.profile);
 
-        DrvInfos installedElems = queryInstalled(*globals.state, globals.profile);
+        PackageInfos installedElems = queryInstalled(*globals.state, globals.profile);
 
         /* Fetch all derivations from the input file. */
-        DrvInfos availElems;
+        PackageInfos availElems;
         queryInstSources(*globals.state, globals.instSource, args, availElems, false);
 
         /* Go through all installed derivations. */
-        DrvInfos newElems;
+        PackageInfos newElems;
         for (auto & i : installedElems) {
             DrvName drvName(i.queryName());
 
@@ -617,7 +617,7 @@ static void upgradeDerivations(Globals & globals,
                    priority.  If there are still multiple matches,
                    take the one with the highest version.
                    Do not upgrade if it would decrease the priority. */
-                DrvInfos::iterator bestElem = availElems.end();
+                PackageInfos::iterator bestElem = availElems.end();
                 std::string bestVersion;
                 for (auto j = availElems.begin(); j != availElems.end(); ++j) {
                     if (comparePriorities(*globals.state, i, *j) > 0)
@@ -687,7 +687,7 @@ static void opUpgrade(Globals & globals, Strings opFlags, Strings opArgs)
 }
 
 
-static void setMetaFlag(EvalState & state, DrvInfo & drv,
+static void setMetaFlag(EvalState & state, PackageInfo & drv,
     const std::string & name, const std::string & value)
 {
     auto v = state.allocValue();
@@ -711,7 +711,7 @@ static void opSetFlag(Globals & globals, Strings opFlags, Strings opArgs)
     while (true) {
         std::string lockToken = optimisticLockProfile(globals.profile);
 
-        DrvInfos installedElems = queryInstalled(*globals.state, globals.profile);
+        PackageInfos installedElems = queryInstalled(*globals.state, globals.profile);
 
         /* Update all matching derivations. */
         for (auto & i : installedElems) {
@@ -745,13 +745,13 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs)
         else throw UsageError("unknown flag '%1%'", arg);
     }
 
-    DrvInfos elems;
+    PackageInfos elems;
     queryInstSources(*globals.state, globals.instSource, opArgs, elems, true);
 
     if (elems.size() != 1)
         throw Error("--set requires exactly one derivation");
 
-    DrvInfo & drv(elems.front());
+    PackageInfo & drv(elems.front());
 
     if (globals.forceName != "")
         drv.setName(globals.forceName);
@@ -786,10 +786,10 @@ static void uninstallDerivations(Globals & globals, Strings & selectors,
     while (true) {
         auto lockToken = optimisticLockProfile(profile);
 
-        DrvInfos workingElems = queryInstalled(*globals.state, profile);
+        PackageInfos workingElems = queryInstalled(*globals.state, profile);
 
         for (auto & selector : selectors) {
-            DrvInfos::iterator split = workingElems.begin();
+            PackageInfos::iterator split = workingElems.begin();
             if (isPath(selector)) {
                 StorePath selectorStorePath = globals.state->store->followLinksToStorePath(selector);
                 split = std::partition(
@@ -838,7 +838,7 @@ static bool cmpChars(char a, char b)
 }
 
 
-static bool cmpElemByName(const DrvInfo & a, const DrvInfo & b)
+static bool cmpElemByName(const PackageInfo & a, const PackageInfo & b)
 {
     auto a_name = a.queryName();
     auto b_name = b.queryName();
@@ -891,7 +891,7 @@ void printTable(Table & table)
 typedef enum { cvLess, cvEqual, cvGreater, cvUnavail } VersionDiff;
 
 static VersionDiff compareVersionAgainstSet(
-    const DrvInfo & elem, const DrvInfos & elems, std::string & version)
+    const PackageInfo & elem, const PackageInfos & elems, std::string & version)
 {
     DrvName name(elem.queryName());
 
@@ -922,7 +922,7 @@ static VersionDiff compareVersionAgainstSet(
 }
 
 
-static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool printOutPath, bool printDrvPath, bool printMeta)
+static void queryJSON(Globals & globals, std::vector<PackageInfo> & elems, bool printOutPath, bool printDrvPath, bool printMeta)
 {
     using nlohmann::json;
     json topObj = json::object();
@@ -942,7 +942,7 @@ static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool prin
             };
 
             {
-                DrvInfo::Outputs outputs = i.queryOutputs(printOutPath);
+                PackageInfo::Outputs outputs = i.queryOutputs(printOutPath);
                 json &outputObj = pkgObj["outputs"];
                 outputObj = json::object();
                 for (auto & j : outputs) {
@@ -1032,7 +1032,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
         throw UsageError("--attr-path(-P) only works with --available");
 
     /* Obtain derivation information from the specified source. */
-    DrvInfos availElems, installedElems;
+    PackageInfos availElems, installedElems;
 
     if (source == sInstalled || compareVersions || printStatus)
         installedElems = queryInstalled(*globals.state, globals.profile);
@@ -1042,16 +1042,16 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
             globals.instSource.systemFilter, *globals.instSource.autoArgs,
             attrPath, availElems);
 
-    DrvInfos elems_ = filterBySelector(*globals.state,
+    PackageInfos elems_ = filterBySelector(*globals.state,
         source == sInstalled ? installedElems : availElems,
         opArgs, false);
 
-    DrvInfos & otherElems(source == sInstalled ? availElems : installedElems);
+    PackageInfos & otherElems(source == sInstalled ? availElems : installedElems);
 
 
     /* Sort them by name. */
     /* !!! */
-    std::vector<DrvInfo> elems;
+    std::vector<PackageInfo> elems;
     for (auto & i : elems_) elems.push_back(i);
     sort(elems.begin(), elems.end(), cmpElemByName);
 
@@ -1192,7 +1192,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
                 attrs["outputName"] = i.queryOutputName();
 
             if (printOutPath && !xmlOutput) {
-                DrvInfo::Outputs outputs = i.queryOutputs();
+                PackageInfo::Outputs outputs = i.queryOutputs();
                 std::string s;
                 for (auto & j : outputs) {
                     if (!s.empty()) s += ';';
@@ -1212,7 +1212,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
 
             if (xmlOutput) {
                 XMLOpenElement item(xml, "item", attrs);
-                DrvInfo::Outputs outputs = i.queryOutputs(printOutPath);
+                PackageInfo::Outputs outputs = i.queryOutputs(printOutPath);
                 for (auto & j : outputs) {
                     XMLAttrs attrs2;
                     attrs2["name"] = j.first;
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index 973b6ee2b..2f9c988d5 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -15,9 +15,9 @@
 namespace nix {
 
 
-DrvInfos queryInstalled(EvalState & state, const Path & userEnv)
+PackageInfos queryInstalled(EvalState & state, const Path & userEnv)
 {
-    DrvInfos elems;
+    PackageInfos elems;
     if (pathExists(userEnv + "/manifest.json"))
         throw Error("profile '%s' is incompatible with 'nix-env'; please use 'nix profile' instead", userEnv);
     auto manifestFile = userEnv + "/manifest.nix";
@@ -31,7 +31,7 @@ DrvInfos queryInstalled(EvalState & state, const Path & userEnv)
 }
 
 
-bool createUserEnv(EvalState & state, DrvInfos & elems,
+bool createUserEnv(EvalState & state, PackageInfos & elems,
     const Path & profile, bool keepDerivations,
     const std::string & lockToken)
 {
@@ -57,7 +57,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
            output paths, and optionally the derivation path, as well
            as the meta attributes. */
         std::optional<StorePath> drvPath = keepDerivations ? i.queryDrvPath() : std::nullopt;
-        DrvInfo::Outputs outputs = i.queryOutputs(true, true);
+        PackageInfo::Outputs outputs = i.queryOutputs(true, true);
         StringSet metaNames = i.queryMetaNames();
 
         auto attrs = state.buildBindings(7 + outputs.size());
diff --git a/src/nix-env/user-env.hh b/src/nix-env/user-env.hh
index af45d2d85..15da3fcb3 100644
--- a/src/nix-env/user-env.hh
+++ b/src/nix-env/user-env.hh
@@ -5,9 +5,9 @@
 
 namespace nix {
 
-DrvInfos queryInstalled(EvalState & state, const Path & userEnv);
+PackageInfos queryInstalled(EvalState & state, const Path & userEnv);
 
-bool createUserEnv(EvalState & state, DrvInfos & elems,
+bool createUserEnv(EvalState & state, PackageInfos & elems,
     const Path & profile, bool keepDerivations,
     const std::string & lockToken);
 
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index 87bc986e8..b9e626aed 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -62,7 +62,7 @@ void processExpr(EvalState & state, const Strings & attrPaths,
                 std::cout << std::endl;
             }
         } else {
-            DrvInfos drvs;
+            PackageInfos drvs;
             getDerivations(state, v, "", autoArgs, drvs, false);
             for (auto & i : drvs) {
                 auto drvPath = i.requireDrvPath();
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 2b6e56283..bebc62deb 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -395,11 +395,11 @@ struct CmdFlakeCheck : FlakeCommand
 
         auto checkDerivation = [&](const std::string & attrPath, Value & v, const PosIdx pos) -> std::optional<StorePath> {
             try {
-                auto drvInfo = getDerivation(*state, v, false);
-                if (!drvInfo)
+                auto packageInfo = getDerivation(*state, v, false);
+                if (!packageInfo)
                     throw Error("flake attribute '%s' is not a derivation", attrPath);
                 // FIXME: check meta attributes
-                return drvInfo->queryDrvPath();
+                return packageInfo->queryDrvPath();
             } catch (Error & e) {
                 e.addTrace(resolve(pos), hintfmt("while checking the derivation '%s'", attrPath));
                 reportError(e);
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 616fe9512..812e703b4 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -168,11 +168,11 @@ struct ProfileManifest
             state.allowPath(state.store->followLinksToStore(profile));
             state.allowPath(state.store->followLinksToStore(profile + "/manifest.nix"));
 
-            auto drvInfos = queryInstalled(state, state.store->followLinksToStore(profile));
+            auto packageInfos = queryInstalled(state, state.store->followLinksToStore(profile));
 
-            for (auto & drvInfo : drvInfos) {
+            for (auto & packageInfo : packageInfos) {
                 ProfileElement element;
-                element.storePaths = {drvInfo.queryOutPath()};
+                element.storePaths = {packageInfo.queryOutPath()};
                 addElement(std::move(element));
             }
         }

From ea6aa5ffd87b27ddd89cab541f4b98b3efcb7ea9 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Tue, 16 Jan 2024 15:44:02 +0100
Subject: [PATCH 382/421] Package{,Info}: comments

---
 src/libexpr/get-drvs.hh           | 4 +++-
 src/libstore/builtins/buildenv.hh | 3 +++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh
index b886581b6..e8c1190f7 100644
--- a/src/libexpr/get-drvs.hh
+++ b/src/libexpr/get-drvs.hh
@@ -10,7 +10,9 @@
 
 namespace nix {
 
-
+/**
+ * A "parsed" package attribute set.
+ */
 struct PackageInfo
 {
 public:
diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh
index 8bebd390d..b24633e27 100644
--- a/src/libstore/builtins/buildenv.hh
+++ b/src/libstore/builtins/buildenv.hh
@@ -5,6 +5,9 @@
 
 namespace nix {
 
+/**
+ * Think of this as a "store level package attrset", but stripped down to no more than the needs of buildenv.
+ */
 struct Package {
     Path path;
     bool active;

From 0bc66e529fa34b84ae31301dd99f31cc16ccfd6c Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Thu, 2 Nov 2023 10:13:55 +0100
Subject: [PATCH 383/421] Use npos member variables instead of full type

---
 src/libutil/file-system.cc |  6 +++---
 src/libutil/util.cc        | 10 +++++-----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc
index ab8d32275..14d496958 100644
--- a/src/libutil/file-system.cc
+++ b/src/libutil/file-system.cc
@@ -90,7 +90,7 @@ Path canonPath(PathView path, bool resolveSymlinks)
         /* Normal component; copy it. */
         else {
             s += '/';
-            if (const auto slash = path.find('/'); slash == std::string::npos) {
+            if (const auto slash = path.find('/'); slash == path.npos) {
                 s += path;
                 path = {};
             } else {
@@ -123,7 +123,7 @@ Path canonPath(PathView path, bool resolveSymlinks)
 Path dirOf(const PathView path)
 {
     Path::size_type pos = path.rfind('/');
-    if (pos == std::string::npos)
+    if (pos == path.npos)
         return ".";
     return pos == 0 ? "/" : Path(path, 0, pos);
 }
@@ -139,7 +139,7 @@ std::string_view baseNameOf(std::string_view path)
         last -= 1;
 
     auto pos = path.rfind('/', last);
-    if (pos == std::string::npos)
+    if (pos == path.npos)
         pos = 0;
     else
         pos += 1;
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index b23362b5c..6e47ce2a3 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -52,9 +52,9 @@ template<class C> C tokenizeString(std::string_view s, std::string_view separato
 {
     C result;
     auto pos = s.find_first_not_of(separators, 0);
-    while (pos != std::string_view::npos) {
+    while (pos != s.npos) {
         auto end = s.find_first_of(separators, pos + 1);
-        if (end == std::string_view::npos) end = s.size();
+        if (end == s.npos) end = s.size();
         result.insert(result.end(), std::string(s, pos, end - pos));
         pos = s.find_first_not_of(separators, end);
     }
@@ -69,7 +69,7 @@ template std::vector<std::string> tokenizeString(std::string_view s, std::string
 std::string chomp(std::string_view s)
 {
     size_t i = s.find_last_not_of(" \n\r\t");
-    return i == std::string_view::npos ? "" : std::string(s, 0, i + 1);
+    return i == s.npos ? "" : std::string(s, 0, i + 1);
 }
 
 
@@ -89,7 +89,7 @@ std::string replaceStrings(
 {
     if (from.empty()) return res;
     size_t pos = 0;
-    while ((pos = res.find(from, pos)) != std::string::npos) {
+    while ((pos = res.find(from, pos)) != res.npos) {
         res.replace(pos, from.size(), to);
         pos += to.size();
     }
@@ -102,7 +102,7 @@ std::string rewriteStrings(std::string s, const StringMap & rewrites)
     for (auto & i : rewrites) {
         if (i.first == i.second) continue;
         size_t j = 0;
-        while ((j = s.find(i.first, j)) != std::string::npos)
+        while ((j = s.find(i.first, j)) != s.npos)
             s.replace(j, i.first.size(), i.second);
     }
     return s;

From 1885d579db145d45f0aaf6130cd7e4db17b5e214 Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Thu, 2 Nov 2023 15:49:22 +0100
Subject: [PATCH 384/421] Improve String Handling

---
 src/libutil/file-system.cc | 6 +++++-
 src/libutil/util.cc        | 9 ++++-----
 src/libutil/util.hh        | 2 +-
 3 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc
index 14d496958..cf8a6d967 100644
--- a/src/libutil/file-system.cc
+++ b/src/libutil/file-system.cc
@@ -116,7 +116,11 @@ Path canonPath(PathView path, bool resolveSymlinks)
         }
     }
 
-    return s.empty() ? "/" : std::move(s);
+    if (s.empty()) {
+        s = "/";
+    }
+
+    return s;
 }
 
 
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 6e47ce2a3..8f310c6fe 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -122,12 +122,11 @@ bool hasSuffix(std::string_view s, std::string_view suffix)
 }
 
 
-std::string toLower(const std::string & s)
+std::string toLower(std::string s)
 {
-    std::string r(s);
-    for (auto & c : r)
+    for (auto & c : s)
         c = std::tolower(c);
-    return r;
+    return s;
 }
 
 
@@ -135,7 +134,7 @@ std::string shellEscape(const std::string_view s)
 {
     std::string r;
     r.reserve(s.size() + 2);
-    r += "'";
+    r += '\'';
     for (auto & i : s)
         if (i == '\'') r += "'\\''"; else r += i;
     r += '\'';
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 27faa4d6d..11a0431da 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -180,7 +180,7 @@ bool hasSuffix(std::string_view s, std::string_view suffix);
 /**
  * Convert a string to lower case.
  */
-std::string toLower(const std::string & s);
+std::string toLower(std::string s);
 
 
 /**

From c924147c9d782e70e0ad421329252ced57f88d09 Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Thu, 2 Nov 2023 15:50:00 +0100
Subject: [PATCH 385/421] Drop parentheses from thunks

---
 src/libutil/file-descriptor.cc    |  2 +-
 src/libutil/processes.cc          | 12 ++++++------
 src/libutil/unix-domain-socket.cc |  4 ++--
 src/libutil/util.cc               |  2 +-
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc
index 692be3383..43e3cd979 100644
--- a/src/libutil/file-descriptor.cc
+++ b/src/libutil/file-descriptor.cc
@@ -96,7 +96,7 @@ void drainFD(int fd, Sink & sink, bool block)
             throw SysError("making file descriptor non-blocking");
     }
 
-    Finally finally([&]() {
+    Finally finally([&] {
         if (!block) {
             if (fcntl(fd, F_SETFL, saved) == -1)
                 throw SysError("making file descriptor blocking");
diff --git a/src/libutil/processes.cc b/src/libutil/processes.cc
index 91a0ea66f..e1e60302b 100644
--- a/src/libutil/processes.cc
+++ b/src/libutil/processes.cc
@@ -131,7 +131,7 @@ void killUser(uid_t uid)
        users to which the current process can send signals.  So we
        fork a process, switch to uid, and send a mass kill. */
 
-    Pid pid = startProcess([&]() {
+    Pid pid = startProcess([&] {
 
         if (setuid(uid) == -1)
             throw SysError("setting uid");
@@ -197,7 +197,7 @@ static int childEntry(void * arg)
 
 pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
 {
-    std::function<void()> wrapper = [&]() {
+    ChildWrapperFunction wrapper = [&] {
         if (!options.allowVfork)
             logger = makeSimpleLogger();
         try {
@@ -229,7 +229,7 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
             PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
         if (stack == MAP_FAILED) throw SysError("allocating stack");
 
-        Finally freeStack([&]() { munmap(stack, stackSize); });
+        Finally freeStack([&] { munmap(stack, stackSize); });
 
         pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper);
         #else
@@ -308,7 +308,7 @@ void runProgram2(const RunOptions & options)
     }
 
     /* Fork. */
-    Pid pid = startProcess([&]() {
+    Pid pid = startProcess([&] {
         if (options.environment)
             replaceEnv(*options.environment);
         if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
@@ -350,7 +350,7 @@ void runProgram2(const RunOptions & options)
 
     std::promise<void> promise;
 
-    Finally doJoin([&]() {
+    Finally doJoin([&] {
         if (writerThread.joinable())
             writerThread.join();
     });
@@ -358,7 +358,7 @@ void runProgram2(const RunOptions & options)
 
     if (source) {
         in.readSide.close();
-        writerThread = std::thread([&]() {
+        writerThread = std::thread([&] {
             try {
                 std::vector<char> buf(8 * 1024);
                 while (true) {
diff --git a/src/libutil/unix-domain-socket.cc b/src/libutil/unix-domain-socket.cc
index 05bbb5ba3..dc19daf9e 100644
--- a/src/libutil/unix-domain-socket.cc
+++ b/src/libutil/unix-domain-socket.cc
@@ -47,7 +47,7 @@ void bind(int fd, const std::string & path)
     addr.sun_family = AF_UNIX;
 
     if (path.size() + 1 >= sizeof(addr.sun_path)) {
-        Pid pid = startProcess([&]() {
+        Pid pid = startProcess([&] {
             Path dir = dirOf(path);
             if (chdir(dir.c_str()) == -1)
                 throw SysError("chdir to '%s' failed", dir);
@@ -78,7 +78,7 @@ void connect(int fd, const std::string & path)
     if (path.size() + 1 >= sizeof(addr.sun_path)) {
         Pipe pipe;
         pipe.create();
-        Pid pid = startProcess([&]() {
+        Pid pid = startProcess([&] {
             try {
                 pipe.readSide.close();
                 Path dir = dirOf(path);
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 8f310c6fe..75bb31c9b 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -183,7 +183,7 @@ std::string base64Encode(std::string_view s)
 std::string base64Decode(std::string_view s)
 {
     constexpr char npos = -1;
-    constexpr std::array<char, 256> base64DecodeChars = [&]() {
+    constexpr std::array<char, 256> base64DecodeChars = [&] {
         std::array<char, 256>  result{};
         for (auto& c : result)
             c = npos;

From d11d7849f7676eb8b2c771356b9be8d8bb756cc8 Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Thu, 2 Nov 2023 15:51:47 +0100
Subject: [PATCH 386/421] Use ChildWrapperFunction type and make casts more
 explicit

---
 src/libutil/processes.cc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/libutil/processes.cc b/src/libutil/processes.cc
index e1e60302b..28f1adcf0 100644
--- a/src/libutil/processes.cc
+++ b/src/libutil/processes.cc
@@ -168,11 +168,12 @@ void killUser(uid_t uid)
 
 //////////////////////////////////////////////////////////////////////
 
+using ChildWrapperFunction = std::function<void()>;
 
 /* Wrapper around vfork to prevent the child process from clobbering
    the caller's stack frame in the parent. */
-static pid_t doFork(bool allowVfork, std::function<void()> fun) __attribute__((noinline));
-static pid_t doFork(bool allowVfork, std::function<void()> fun)
+static pid_t doFork(bool allowVfork, ChildWrapperFunction & fun) __attribute__((noinline));
+static pid_t doFork(bool allowVfork, ChildWrapperFunction & fun)
 {
 #ifdef __linux__
     pid_t pid = allowVfork ? vfork() : fork();
@@ -188,8 +189,8 @@ static pid_t doFork(bool allowVfork, std::function<void()> fun)
 #if __linux__
 static int childEntry(void * arg)
 {
-    auto main = (std::function<void()> *) arg;
-    (*main)();
+    auto & fun = *reinterpret_cast<ChildWrapperFunction*>(arg);
+    fun();
     return 1;
 }
 #endif

From 9d9f42cc38b06ddc3fe30f4c1695514774b5217e Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Thu, 2 Nov 2023 15:52:38 +0100
Subject: [PATCH 387/421] Remove C-style casts

---
 src/libutil/file-descriptor.cc    |  2 +-
 src/libutil/processes.cc          |  4 ++--
 src/libutil/unix-domain-socket.cc | 18 ++++++++++++++----
 3 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc
index 43e3cd979..55d57e29b 100644
--- a/src/libutil/file-descriptor.cc
+++ b/src/libutil/file-descriptor.cc
@@ -114,7 +114,7 @@ void drainFD(int fd, Sink & sink, bool block)
                 throw SysError("reading from file");
         }
         else if (rd == 0) break;
-        else sink({(char *) buf.data(), (size_t) rd});
+        else sink({reinterpret_cast<char *>(buf.data()), size_t(rd)});
     }
 }
 
diff --git a/src/libutil/processes.cc b/src/libutil/processes.cc
index 28f1adcf0..f5d584330 100644
--- a/src/libutil/processes.cc
+++ b/src/libutil/processes.cc
@@ -226,8 +226,8 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
         assert(!(options.cloneFlags & CLONE_VM));
 
         size_t stackSize = 1 * 1024 * 1024;
-        auto stack = (char *) mmap(0, stackSize,
-            PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
+        auto stack = static_cast<char *>(mmap(0, stackSize,
+            PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0));
         if (stack == MAP_FAILED) throw SysError("allocating stack");
 
         Finally freeStack([&] { munmap(stack, stackSize); });
diff --git a/src/libutil/unix-domain-socket.cc b/src/libutil/unix-domain-socket.cc
index dc19daf9e..3b6d54a2c 100644
--- a/src/libutil/unix-domain-socket.cc
+++ b/src/libutil/unix-domain-socket.cc
@@ -38,6 +38,14 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
     return fdSocket;
 }
 
+static struct sockaddr* safeSockAddrPointerCast(struct sockaddr_un *addr) {
+    // Casting between types like these legacy C library interfaces require
+    // is forbidden in C++.
+    // To maintain backwards compatibility, the implementation of the
+    // bind function contains some hints to the compiler that allow for this
+    // special case.
+    return reinterpret_cast<struct sockaddr *>(addr);
+}
 
 void bind(int fd, const std::string & path)
 {
@@ -45,6 +53,7 @@ void bind(int fd, const std::string & path)
 
     struct sockaddr_un addr;
     addr.sun_family = AF_UNIX;
+    auto psaddr {safeSockAddrPointerCast(&addr)};
 
     if (path.size() + 1 >= sizeof(addr.sun_path)) {
         Pid pid = startProcess([&] {
@@ -55,7 +64,7 @@ void bind(int fd, const std::string & path)
             if (base.size() + 1 >= sizeof(addr.sun_path))
                 throw Error("socket path '%s' is too long", base);
             memcpy(addr.sun_path, base.c_str(), base.size() + 1);
-            if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+            if (bind(fd, psaddr, sizeof(addr)) == -1)
                 throw SysError("cannot bind to socket '%s'", path);
             _exit(0);
         });
@@ -64,7 +73,7 @@ void bind(int fd, const std::string & path)
             throw Error("cannot bind to socket '%s'", path);
     } else {
         memcpy(addr.sun_path, path.c_str(), path.size() + 1);
-        if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+        if (bind(fd, psaddr, sizeof(addr)) == -1)
             throw SysError("cannot bind to socket '%s'", path);
     }
 }
@@ -74,6 +83,7 @@ void connect(int fd, const std::string & path)
 {
     struct sockaddr_un addr;
     addr.sun_family = AF_UNIX;
+    auto psaddr {safeSockAddrPointerCast(&addr)};
 
     if (path.size() + 1 >= sizeof(addr.sun_path)) {
         Pipe pipe;
@@ -88,7 +98,7 @@ void connect(int fd, const std::string & path)
                 if (base.size() + 1 >= sizeof(addr.sun_path))
                     throw Error("socket path '%s' is too long", base);
                 memcpy(addr.sun_path, base.c_str(), base.size() + 1);
-                if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+                if (connect(fd, psaddr, sizeof(addr)) == -1)
                     throw SysError("cannot connect to socket at '%s'", path);
                 writeFull(pipe.writeSide.get(), "0\n");
             } catch (SysError & e) {
@@ -107,7 +117,7 @@ void connect(int fd, const std::string & path)
         }
     } else {
         memcpy(addr.sun_path, path.c_str(), path.size() + 1);
-        if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+        if (connect(fd, psaddr, sizeof(addr)) == -1)
             throw SysError("cannot connect to socket at '%s'", path);
     }
 }

From 8ae3aeec9442e2b249abdb42a2853618b74a68a2 Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek@galowicz.de>
Date: Thu, 2 Nov 2023 15:52:53 +0100
Subject: [PATCH 388/421] Don't use std::make_unique right before release

---
 src/libutil/signals.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libutil/signals.cc b/src/libutil/signals.cc
index 4632aa319..eaa4ea30e 100644
--- a/src/libutil/signals.cc
+++ b/src/libutil/signals.cc
@@ -179,7 +179,7 @@ std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()>
     auto token = interruptCallbacks->nextToken++;
     interruptCallbacks->callbacks.emplace(token, callback);
 
-    auto res = std::make_unique<InterruptCallbackImpl>();
+    std::unique_ptr<InterruptCallbackImpl> res {new InterruptCallbackImpl{}};
     res->token = token;
 
     return std::unique_ptr<InterruptCallback>(res.release());

From 55da93942428d51ba3fa3577d3ff79cd739fb38e Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Wed, 17 Jan 2024 02:57:11 +0100
Subject: [PATCH 389/421] fix typo

---
 doc/manual/src/contributing/documentation.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/contributing/documentation.md
index 75226cd1a..1dddb207c 100644
--- a/doc/manual/src/contributing/documentation.md
+++ b/doc/manual/src/contributing/documentation.md
@@ -172,7 +172,7 @@ Please observe these guidelines to ease reviews:
   > ```
   ````
 
-  Highlight syntax definiions as such, using [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) notation:
+  Highlight syntax definitions as such, using [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) notation:
 
   ````
   > **Syntax**

From f134dbdffb81cea72a2e4abfb9a13904417b82aa Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Wed, 17 Jan 2024 04:23:16 +0100
Subject: [PATCH 390/421] move section on make variables

it should be after the general build instructions, as it goes into more detail.
---
 doc/manual/src/contributing/hacking.md | 42 +++++++++++++-------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index 0fa59e891..fbdc7b7f7 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -67,27 +67,6 @@ $ nix build
 
 You can also build Nix for one of the [supported platforms](#platforms).
 
-## Makefile variables
-
-You may need `profiledir=$out/etc/profile.d` and `sysconfdir=$out/etc` to run
-`make install`.
-
-You may want to set `MAKEFLAGS="-e -j $NIX_BUILD_CORES"` to allow environment
-variables to override `Makefile` variables.
-
-- `ENABLE_BUILD=yes` to enable building the C++ code.
-- `ENABLE_DOC_GEN=yes` to enable building the documentation (manual, man pages, etc.).
-
-  The docs can take a while to build, so you may want to disable this for local development.
-- `ENABLE_FUNCTIONAL_TESTS=yes` to enable building the functional tests.
-- `ENABLE_UNIT_TESTS=yes` to enable building the unit tests.
-- `OPTIMIZE=1` to enable optimizations.
-- `libraries=libutil programs=` to only build a specific library (this will
-  fail in the linking phase if you don't have the other libraries built, but is
-  useful for checking types).
-- `libraries= programs=nix` to only build a specific program (this will not, in
-  general, work, because the programs need the libraries).
-
 ## Building Nix
 
 To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found:
@@ -132,6 +111,27 @@ $ nix-build
 
 You can also build Nix for one of the [supported platforms](#platforms).
 
+## Makefile variables
+
+You may need `profiledir=$out/etc/profile.d` and `sysconfdir=$out/etc` to run
+`make install`.
+
+You may want to set `MAKEFLAGS="-e -j $NIX_BUILD_CORES"` to allow environment
+variables to override `Makefile` variables.
+
+- `ENABLE_BUILD=yes` to enable building the C++ code.
+- `ENABLE_DOC_GEN=yes` to enable building the documentation (manual, man pages, etc.).
+
+  The docs can take a while to build, so you may want to disable this for local development.
+- `ENABLE_FUNCTIONAL_TESTS=yes` to enable building the functional tests.
+- `ENABLE_UNIT_TESTS=yes` to enable building the unit tests.
+- `OPTIMIZE=1` to enable optimizations.
+- `libraries=libutil programs=` to only build a specific library (this will
+  fail in the linking phase if you don't have the other libraries built, but is
+  useful for checking types).
+- `libraries= programs=nix` to only build a specific program (this will not, in
+  general, work, because the programs need the libraries).
+
 ## Platforms
 
 Nix can be built for various platforms, as specified in [`flake.nix`]:

From 28eb406834ed176d84e22898ccbcf4ecb963416c Mon Sep 17 00:00:00 2001
From: Valentin Gagarin <valentin.gagarin@tweag.io>
Date: Wed, 17 Jan 2024 04:39:26 +0100
Subject: [PATCH 391/421] reword section on make variables

- use one line per sentence
- use imperative for instructions
- add link to Make documentation
---
 doc/manual/src/contributing/hacking.md | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index fbdc7b7f7..fe91787a3 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -113,11 +113,9 @@ You can also build Nix for one of the [supported platforms](#platforms).
 
 ## Makefile variables
 
-You may need `profiledir=$out/etc/profile.d` and `sysconfdir=$out/etc` to run
-`make install`.
+You may need `profiledir=$out/etc/profile.d` and `sysconfdir=$out/etc` to run `make install`.
 
-You may want to set `MAKEFLAGS="-e -j $NIX_BUILD_CORES"` to allow environment
-variables to override `Makefile` variables.
+Run `make` with [`--environment-overrides`](https://www.gnu.org/software/make/manual/make.html#index-_002de) to allow environment variables to override `Makefile` variables:
 
 - `ENABLE_BUILD=yes` to enable building the C++ code.
 - `ENABLE_DOC_GEN=yes` to enable building the documentation (manual, man pages, etc.).
@@ -126,11 +124,12 @@ variables to override `Makefile` variables.
 - `ENABLE_FUNCTIONAL_TESTS=yes` to enable building the functional tests.
 - `ENABLE_UNIT_TESTS=yes` to enable building the unit tests.
 - `OPTIMIZE=1` to enable optimizations.
-- `libraries=libutil programs=` to only build a specific library (this will
-  fail in the linking phase if you don't have the other libraries built, but is
-  useful for checking types).
-- `libraries= programs=nix` to only build a specific program (this will not, in
-  general, work, because the programs need the libraries).
+- `libraries=libutil programs=` to only build a specific library.
+
+  This will fail in the linking phase if the other libraries haven't been built, but is useful for checking types.
+- `libraries= programs=nix` to only build a specific program.
+
+  This will not work in general, because the programs need the libraries.
 
 ## Platforms
 

From d0a284284bc93014c98294292b7f4b95864f37ee Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Wed, 17 Jan 2024 16:54:45 +0100
Subject: [PATCH 392/421] refactor: Extract simply, awkwardly
 Store::queryPathInfoFromClientCache

This is useful for determining quickly which substituters to query.

An alternative would be for users to invoke the narinfo cache db directly,
so why do we need this change?

 - It is easier to use. I believe Nix itself should also use it.

 - This way, the narinfo cache db remains an implementation detail.

 - Callers get to use the in-memory cache as well.
---
 src/libstore/store-api.cc | 64 +++++++++++++++++++++++----------------
 src/libstore/store-api.hh | 12 ++++++++
 2 files changed, 50 insertions(+), 26 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 0c37ecd30..66bc95625 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -685,6 +685,42 @@ static bool goodStorePath(const StorePath & expected, const StorePath & actual)
         && (expected.name() == Store::MissingName || expected.name() == actual.name());
 }
 
+bool Store::queryPathInfoFromClientCache(const StorePath & storePath,
+    Callback<ref<const ValidPathInfo>> & callback)
+{
+    auto hashPart = std::string(storePath.hashPart());
+
+    {
+        auto res = state.lock()->pathInfoCache.get(std::string(storePath.to_string()));
+        if (res && res->isKnownNow()) {
+            stats.narInfoReadAverted++;
+            if (!res->didExist())
+                throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
+            callback(ref<const ValidPathInfo>(res->value));
+            return true;
+        }
+    }
+
+    if (diskCache) {
+        auto res = diskCache->lookupNarInfo(getUri(), hashPart);
+        if (res.first != NarInfoDiskCache::oUnknown) {
+            stats.narInfoReadAverted++;
+            {
+                auto state_(state.lock());
+                state_->pathInfoCache.upsert(std::string(storePath.to_string()),
+                    res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
+                if (res.first == NarInfoDiskCache::oInvalid ||
+                    !goodStorePath(storePath, res.second->path))
+                    throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
+            }
+            callback(ref<const ValidPathInfo>(res.second));
+            return true;
+        }
+    }
+
+    return false;
+}
+
 
 void Store::queryPathInfo(const StorePath & storePath,
     Callback<ref<const ValidPathInfo>> callback) noexcept
@@ -692,32 +728,8 @@ void Store::queryPathInfo(const StorePath & storePath,
     auto hashPart = std::string(storePath.hashPart());
 
     try {
-        {
-            auto res = state.lock()->pathInfoCache.get(std::string(storePath.to_string()));
-            if (res && res->isKnownNow()) {
-                stats.narInfoReadAverted++;
-                if (!res->didExist())
-                    throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
-                return callback(ref<const ValidPathInfo>(res->value));
-            }
-        }
-
-        if (diskCache) {
-            auto res = diskCache->lookupNarInfo(getUri(), hashPart);
-            if (res.first != NarInfoDiskCache::oUnknown) {
-                stats.narInfoReadAverted++;
-                {
-                    auto state_(state.lock());
-                    state_->pathInfoCache.upsert(std::string(storePath.to_string()),
-                        res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
-                    if (res.first == NarInfoDiskCache::oInvalid ||
-                        !goodStorePath(storePath, res.second->path))
-                        throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
-                }
-                return callback(ref<const ValidPathInfo>(res.second));
-            }
-        }
-
+        if (queryPathInfoFromClientCache(storePath, callback))
+            return;
     } catch (...) { return callback.rethrow(); }
 
     auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 9667b5e9e..2a1092d9e 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -282,6 +282,18 @@ public:
     void queryPathInfo(const StorePath & path,
         Callback<ref<const ValidPathInfo>> callback) noexcept;
 
+    /**
+     * NOTE: this is not the final interface - to be modified in next commit.
+     *
+     * Asynchronous version that only queries the local narinfo cache and not
+     * the actual store.
+     *
+     * @return true if the path was known and the callback invoked
+     * @return false if the path was not known and the callback not invoked
+     * @throw InvalidPathError if the path is known to be invalid
+     */
+    bool queryPathInfoFromClientCache(const StorePath & path, Callback<ref<const ValidPathInfo>> & callback);
+
     /**
      * Query the information about a realisation.
      */

From e938912cff5ccded14444377f9776e86c585f917 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 17 Jan 2024 13:08:03 -0500
Subject: [PATCH 393/421] Fix indentation error in `flake.nix`

---
 flake.nix | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/flake.nix b/flake.nix
index 49f214e72..0309f84b3 100644
--- a/flake.nix
+++ b/flake.nix
@@ -197,12 +197,12 @@
               perl-bindings = final.nix-perl-bindings;
             };
 
-            nix-perl-bindings = final.callPackage ./perl {
-              inherit fileset stdenv;
-            };
-
+          nix-perl-bindings = final.callPackage ./perl {
+            inherit fileset stdenv;
           };
 
+        };
+
     in {
       # A Nixpkgs overlay that overrides the 'nix' and
       # 'nix.perl-bindings' packages.

From 1de8eed28a3cb1e449c10ecdb524aec27fe9dc35 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 17 Jan 2024 13:11:04 -0500
Subject: [PATCH 394/421] Move dependency patches from top level into subdir

Good to not clutter the top-level directory.
---
 .../boehmgc-coroutine-sp-fallback.diff                        | 0
 .../boehmgc-traceable_allocator-public.diff                   | 0
 flake.nix                                                     | 4 ++--
 package.nix                                                   | 1 -
 4 files changed, 2 insertions(+), 3 deletions(-)
 rename boehmgc-coroutine-sp-fallback.diff => dep-patches/boehmgc-coroutine-sp-fallback.diff (100%)
 rename boehmgc-traceable_allocator-public.diff => dep-patches/boehmgc-traceable_allocator-public.diff (100%)

diff --git a/boehmgc-coroutine-sp-fallback.diff b/dep-patches/boehmgc-coroutine-sp-fallback.diff
similarity index 100%
rename from boehmgc-coroutine-sp-fallback.diff
rename to dep-patches/boehmgc-coroutine-sp-fallback.diff
diff --git a/boehmgc-traceable_allocator-public.diff b/dep-patches/boehmgc-traceable_allocator-public.diff
similarity index 100%
rename from boehmgc-traceable_allocator-public.diff
rename to dep-patches/boehmgc-traceable_allocator-public.diff
diff --git a/flake.nix b/flake.nix
index 49f214e72..2b9ce17e3 100644
--- a/flake.nix
+++ b/flake.nix
@@ -163,10 +163,10 @@
             enableLargeConfig = true;
           }).overrideAttrs(o: {
             patches = (o.patches or []) ++ [
-              ./boehmgc-coroutine-sp-fallback.diff
+              ./dep-patches/boehmgc-coroutine-sp-fallback.diff
 
               # https://github.com/ivmai/bdwgc/pull/586
-              ./boehmgc-traceable_allocator-public.diff
+              ./dep-patches/boehmgc-traceable_allocator-public.diff
             ];
           });
 
diff --git a/package.nix b/package.nix
index a1188ba9c..192df90ab 100644
--- a/package.nix
+++ b/package.nix
@@ -174,7 +174,6 @@ in {
           ./mk
           (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
         ] ++ lib.optionals doBuild [
-          ./boehmgc-coroutine-sp-fallback.diff
           ./doc
           ./misc
           ./precompiled-headers.h

From 39ab50f9ee64f0455e37a8136638d9757252c226 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 17 Jan 2024 21:41:37 +0100
Subject: [PATCH 395/421] Store::buildPaths(): Fix display of store paths

This was broken in 7ac39ff05c8353c665174e8df61dd76a2b0b93db.
---
 src/libstore/build/entry-points.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc
index 74eca63f3..7f0a05d5d 100644
--- a/src/libstore/build/entry-points.cc
+++ b/src/libstore/build/entry-points.cc
@@ -26,9 +26,9 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
         }
         if (i->exitCode != Goal::ecSuccess) {
             if (auto i2 = dynamic_cast<DerivationGoal *>(i.get()))
-                failed.insert(std::string { i2->drvPath.to_string() });
+                failed.insert(printStorePath(i2->drvPath));
             else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
-                failed.insert(std::string { i2->storePath.to_string()});
+                failed.insert(printStorePath(i2->storePath));
         }
     }
 

From a3cf27ca47328b11173147ca7180e0bae798bb2c Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 17 Jan 2024 22:19:51 +0100
Subject: [PATCH 396/421] Print a more helpful message if the daemon crashes

Instead of

   error: unexpected end-of-file

you now get

   error: Nix daemon disconnected unexpectedly (maybe it crashed?)
---
 src/libstore/remote-store.cc | 1 +
 src/libutil/serialise.cc     | 2 +-
 src/libutil/serialise.hh     | 5 +++--
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 078b9fe00..ccf95beef 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -67,6 +67,7 @@ void RemoteStore::initConnection(Connection & conn)
 {
     /* Send the magic greeting, check for the reply. */
     try {
+        conn.from.endOfFileError = "Nix daemon disconnected unexpectedly (maybe it crashed?)";
         conn.to << WORKER_MAGIC_1;
         conn.to.flush();
         StringSink saved;
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 316105603..afbf66b9d 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -132,7 +132,7 @@ size_t FdSource::readUnbuffered(char * data, size_t len)
         n = ::read(fd, data, len);
     } while (n == -1 && errno == EINTR);
     if (n == -1) { _good = false; throw SysError("reading from file"); }
-    if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); }
+    if (n == 0) { _good = false; throw EndOfFile(endOfFileError); }
     read += n;
     return n;
 }
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 3f57ce88b..689b2070b 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -153,12 +153,13 @@ struct FdSource : BufferedSource
 {
     int fd;
     size_t read = 0;
+    std::string endOfFileError{"unexpected end-of-file"};
 
     FdSource() : fd(-1) { }
     FdSource(int fd) : fd(fd) { }
-    FdSource(FdSource&&) = default;
+    FdSource(FdSource &&) = default;
 
-    FdSource& operator=(FdSource && s)
+    FdSource & operator=(FdSource && s)
     {
         fd = s.fd;
         s.fd = -1;

From 3016e67c21c8ea1f1c44528c7895fad1761406c3 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Tue, 16 Jan 2024 10:35:16 -0500
Subject: [PATCH 397/421] `bind`: give same treatment as `connect` in #8544,
 dedup

It is good to propagate the underlying error so whether or not we use a
process to deal with path length issues is not observable.

Also, as these wrapper functions got more and more complex, the code
duplication got worse and worse. The new `bindConnectProcHelper`
function deduplicates them.
---
 src/libutil/unix-domain-socket.cc | 84 ++++++++++++-------------------
 1 file changed, 33 insertions(+), 51 deletions(-)

diff --git a/src/libutil/unix-domain-socket.cc b/src/libutil/unix-domain-socket.cc
index 3b6d54a2c..0bcf9040d 100644
--- a/src/libutil/unix-domain-socket.cc
+++ b/src/libutil/unix-domain-socket.cc
@@ -38,52 +38,20 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
     return fdSocket;
 }
 
-static struct sockaddr* safeSockAddrPointerCast(struct sockaddr_un *addr) {
-    // Casting between types like these legacy C library interfaces require
-    // is forbidden in C++.
-    // To maintain backwards compatibility, the implementation of the
-    // bind function contains some hints to the compiler that allow for this
+
+static void bindConnectProcHelper(
+    std::string_view operationName, auto && operation,
+    int fd, const std::string & path)
+{
+    struct sockaddr_un addr;
+    addr.sun_family = AF_UNIX;
+
+    // Casting between types like these legacy C library interfaces
+    // require is forbidden in C++. To maintain backwards
+    // compatibility, the implementation of the bind/connect functions
+    // contains some hints to the compiler that allow for this
     // special case.
-    return reinterpret_cast<struct sockaddr *>(addr);
-}
-
-void bind(int fd, const std::string & path)
-{
-    unlink(path.c_str());
-
-    struct sockaddr_un addr;
-    addr.sun_family = AF_UNIX;
-    auto psaddr {safeSockAddrPointerCast(&addr)};
-
-    if (path.size() + 1 >= sizeof(addr.sun_path)) {
-        Pid pid = startProcess([&] {
-            Path dir = dirOf(path);
-            if (chdir(dir.c_str()) == -1)
-                throw SysError("chdir to '%s' failed", dir);
-            std::string base(baseNameOf(path));
-            if (base.size() + 1 >= sizeof(addr.sun_path))
-                throw Error("socket path '%s' is too long", base);
-            memcpy(addr.sun_path, base.c_str(), base.size() + 1);
-            if (bind(fd, psaddr, sizeof(addr)) == -1)
-                throw SysError("cannot bind to socket '%s'", path);
-            _exit(0);
-        });
-        int status = pid.wait();
-        if (status != 0)
-            throw Error("cannot bind to socket '%s'", path);
-    } else {
-        memcpy(addr.sun_path, path.c_str(), path.size() + 1);
-        if (bind(fd, psaddr, sizeof(addr)) == -1)
-            throw SysError("cannot bind to socket '%s'", path);
-    }
-}
-
-
-void connect(int fd, const std::string & path)
-{
-    struct sockaddr_un addr;
-    addr.sun_family = AF_UNIX;
-    auto psaddr {safeSockAddrPointerCast(&addr)};
+    auto * psaddr = reinterpret_cast<struct sockaddr *>(&addr);
 
     if (path.size() + 1 >= sizeof(addr.sun_path)) {
         Pipe pipe;
@@ -98,8 +66,8 @@ void connect(int fd, const std::string & path)
                 if (base.size() + 1 >= sizeof(addr.sun_path))
                     throw Error("socket path '%s' is too long", base);
                 memcpy(addr.sun_path, base.c_str(), base.size() + 1);
-                if (connect(fd, psaddr, sizeof(addr)) == -1)
-                    throw SysError("cannot connect to socket at '%s'", path);
+                if (operation(fd, psaddr, sizeof(addr)) == -1)
+                    throw SysError("cannot %s to socket at '%s'", operationName, path);
                 writeFull(pipe.writeSide.get(), "0\n");
             } catch (SysError & e) {
                 writeFull(pipe.writeSide.get(), fmt("%d\n", e.errNo));
@@ -110,16 +78,30 @@ void connect(int fd, const std::string & path)
         pipe.writeSide.close();
         auto errNo = string2Int<int>(chomp(drainFD(pipe.readSide.get())));
         if (!errNo || *errNo == -1)
-            throw Error("cannot connect to socket at '%s'", path);
+            throw Error("cannot %s to socket at '%s'", operationName, path);
         else if (*errNo > 0) {
             errno = *errNo;
-            throw SysError("cannot connect to socket at '%s'", path);
+            throw SysError("cannot %s to socket at '%s'", operationName, path);
         }
     } else {
         memcpy(addr.sun_path, path.c_str(), path.size() + 1);
-        if (connect(fd, psaddr, sizeof(addr)) == -1)
-            throw SysError("cannot connect to socket at '%s'", path);
+        if (operation(fd, psaddr, sizeof(addr)) == -1)
+            throw SysError("cannot %s to socket at '%s'", operationName, path);
     }
 }
 
+
+void bind(int fd, const std::string & path)
+{
+    unlink(path.c_str());
+
+    bindConnectProcHelper("bind", ::bind, fd, path);
+}
+
+
+void connect(int fd, const std::string & path)
+{
+    bindConnectProcHelper("connect", ::connect, fd, path);
+}
+
 }

From 574db8350491d8da3f65625de1f91bc667e67360 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 17 Jan 2024 23:46:03 -0500
Subject: [PATCH 398/421] Push `addToStoreFromDump` `unsupported(...)` down
 `Store` class hierarchy

Instead of having it be the default method in `Store` itself, have it be
the implementation in `DummyStore` and `LegacySSHStore`. Then just the
implementations which fail to provide the method pay the "penalty" of
dealing with the icky `unimplemented` function for non-compliance.

Picks up where #8217. Getting close to no `unsupported` in the `Store`
interface itself!

More progress on issue #5729.
---
 src/libstore/dummy-store.cc      | 9 +++++++++
 src/libstore/legacy-ssh-store.hh | 9 +++++++++
 src/libstore/store-api.hh        | 3 +--
 3 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc
index f52a309d1..e4f13b8f4 100644
--- a/src/libstore/dummy-store.cc
+++ b/src/libstore/dummy-store.cc
@@ -58,6 +58,15 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
         RepairFlag repair, CheckSigsFlag checkSigs) override
     { unsupported("addToStore"); }
 
+    virtual StorePath addToStoreFromDump(
+        Source & dump,
+        std::string_view name,
+        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
+        const StorePathSet & references = StorePathSet(),
+        RepairFlag repair = NoRepair) override
+    { unsupported("addToStore"); }
+
     void narFromPath(const StorePath & path, Sink & sink) override
     { unsupported("narFromPath"); }
 
diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh
index c5a3ce677..7cee31d66 100644
--- a/src/libstore/legacy-ssh-store.hh
+++ b/src/libstore/legacy-ssh-store.hh
@@ -69,6 +69,15 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
         RepairFlag repair) override
     { unsupported("addToStore"); }
 
+    virtual StorePath addToStoreFromDump(
+        Source & dump,
+        std::string_view name,
+        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
+        const StorePathSet & references = StorePathSet(),
+        RepairFlag repair = NoRepair) override
+    { unsupported("addToStore"); }
+
 private:
 
     void putBuildSettings(Connection & conn);
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 9667b5e9e..b3c935db1 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -466,8 +466,7 @@ public:
         ContentAddressMethod method = FileIngestionMethod::Recursive,
         HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
         const StorePathSet & references = StorePathSet(),
-        RepairFlag repair = NoRepair)
-    { unsupported("addToStoreFromDump"); }
+        RepairFlag repair = NoRepair) = 0;
 
     /**
      * Add a mapping indicating that `deriver!outputName` maps to the output path

From 78074bdea4fcb4403cc7b1e533d1fcbcf51e01a5 Mon Sep 17 00:00:00 2001
From: Robert Hensing <roberth@users.noreply.github.com>
Date: Thu, 18 Jan 2024 12:47:29 +0100
Subject: [PATCH 399/421] tests/nixos/fetch-git: Apply suggestions

---
 tests/nixos/fetch-git/default.nix           | 2 +-
 tests/nixos/fetch-git/testsupport/setup.nix | 3 ---
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/tests/nixos/fetch-git/default.nix b/tests/nixos/fetch-git/default.nix
index 254fecaaf..1d6bcb637 100644
--- a/tests/nixos/fetch-git/default.nix
+++ b/tests/nixos/fetch-git/default.nix
@@ -24,7 +24,7 @@
   testCases =
     map
     (testCaseName: {...}: {
-      imports = ["${./test-cases}/${testCaseName}"];
+      imports = [ (./test-cases + "/${testCaseName}") ];
       # ensures tests are named like their directories they are defined in
       name = testCaseName;
     })
diff --git a/tests/nixos/fetch-git/testsupport/setup.nix b/tests/nixos/fetch-git/testsupport/setup.nix
index 2f74f51f8..8fc8e2e7c 100644
--- a/tests/nixos/fetch-git/testsupport/setup.nix
+++ b/tests/nixos/fetch-git/testsupport/setup.nix
@@ -39,9 +39,6 @@ in
             The name of the test case.
 
             A repository with that name will be set up on the gitea server and locally.
-
-            This name can also be used to execute only a single test case via:
-              `nix build .#hydraJobs.fetch-git.{test-case-name}`
           '';
         };
         options.description = mkOption {

From 1fe8f54bd30fead52d21ae472fb4f0f68a5c6fdd Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 18 Jan 2024 15:27:57 +0100
Subject: [PATCH 400/421] Use BackedStringView

---
 src/libutil/serialise.cc | 2 +-
 src/libutil/serialise.hh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index afbf66b9d..7fc211491 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -132,7 +132,7 @@ size_t FdSource::readUnbuffered(char * data, size_t len)
         n = ::read(fd, data, len);
     } while (n == -1 && errno == EINTR);
     if (n == -1) { _good = false; throw SysError("reading from file"); }
-    if (n == 0) { _good = false; throw EndOfFile(endOfFileError); }
+    if (n == 0) { _good = false; throw EndOfFile(std::string(*endOfFileError)); }
     read += n;
     return n;
 }
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 689b2070b..d9522566f 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -153,7 +153,7 @@ struct FdSource : BufferedSource
 {
     int fd;
     size_t read = 0;
-    std::string endOfFileError{"unexpected end-of-file"};
+    BackedStringView endOfFileError{"unexpected end-of-file"};
 
     FdSource() : fd(-1) { }
     FdSource(int fd) : fd(fd) { }

From ab786e22f16eed0d95123d5698eb71079c312584 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 18 Jan 2024 15:29:54 +0100
Subject: [PATCH 401/421] Show what goal is waiting for a build slot

---
 src/libstore/build/worker.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 974a9f510..d57e22393 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -251,7 +251,7 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
 
 void Worker::waitForBuildSlot(GoalPtr goal)
 {
-    debug("wait for build slot");
+    goal->trace("wait for build slot");
     bool isSubstitutionGoal = goal->jobCategory() == JobCategory::Substitution;
     if ((!isSubstitutionGoal && getNrLocalBuilds() < settings.maxBuildJobs) ||
         (isSubstitutionGoal && getNrSubstitutions() < settings.maxSubstitutionJobs))

From a18d8d688a826ff535b3eeff289ef51db33a413b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 18 Jan 2024 17:01:45 +0100
Subject: [PATCH 402/421] LocalStore::addToStore(): Ignore exceptions from
 parseDump()

In the "discard" case (i.e. when the store path already exists
locally), when we call parseDump() from a Finally and it throws an
exception (e.g. if the download of the NAR fails), Nix crashes:

   terminate called after throwing an instance of 'nix::SubstituteGone'
     what():  error: file 'nar/06br3254rx4gz4cvjzxlv028jrx80zg5i4jr62vjmn416dqihgr7.nar.xz' does not exist in binary cache 'http://localhost'
   Aborted (core dumped)
---
 src/libstore/local-store.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 5a399c8be..07068f8f8 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1049,7 +1049,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
     Finally cleanup = [&]() {
         if (!narRead) {
             NullParseSink sink;
-            parseDump(sink, source);
+            try {
+                parseDump(sink, source);
+            } catch (...) {
+                ignoreException();
+            }
         }
     };
 

From dca0a802405be9798e12ad8be2ec6d227d9a2fa2 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 18 Jan 2024 17:16:34 +0100
Subject: [PATCH 403/421] copyStorePath(): Bail out early if the store path
 already exists

In rare cases (e.g. when using allowSubstitutes = false), it's
possible that we simultaneously have a DerivationGoal *and* a
SubstitutionGoal building the same path. So if a DerivationGoal
already built the path while the SubstitutionGoal was waiting for a
download slot, it saves us a superfluous download to exit early.
---
 src/libstore/store-api.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 0c37ecd30..9cb187e66 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -981,6 +981,11 @@ void copyStorePath(
     RepairFlag repair,
     CheckSigsFlag checkSigs)
 {
+    /* Bail out early (before starting a download from srcStore) if
+       dstStore already has this path. */
+    if (!repair && dstStore.isValidPath(storePath))
+        return;
+
     auto srcUri = srcStore.getUri();
     auto dstUri = dstStore.getUri();
     auto storePathS = srcStore.printStorePath(storePath);

From fd41979d7857f6984b4b7571706a45b16f9c0a5d Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 18 Jan 2024 13:12:59 +0100
Subject: [PATCH 404/421] tests/nixos/fetch-git: Factor out gitea repo module

---
 .../fetch-git/testsupport/gitea-repo.nix      | 51 +++++++++++++++++++
 tests/nixos/fetch-git/testsupport/gitea.nix   |  2 +
 tests/nixos/fetch-git/testsupport/setup.nix   | 44 +++++-----------
 3 files changed, 65 insertions(+), 32 deletions(-)
 create mode 100644 tests/nixos/fetch-git/testsupport/gitea-repo.nix

diff --git a/tests/nixos/fetch-git/testsupport/gitea-repo.nix b/tests/nixos/fetch-git/testsupport/gitea-repo.nix
new file mode 100644
index 000000000..916552bb2
--- /dev/null
+++ b/tests/nixos/fetch-git/testsupport/gitea-repo.nix
@@ -0,0 +1,51 @@
+{ lib, ... }:
+let
+  inherit (lib) mkOption types;
+
+  testCaseExtension = { config, ... }: {
+    setupScript = ''
+      repo = Repo("${config.name}")
+    '';
+  };
+in
+{
+  options = {
+    testCases = mkOption {
+      type = types.listOf (types.submodule testCaseExtension);
+    };
+  };
+  config = {
+    setupScript = ''
+      class Repo:
+        """
+        A class to create a git repository on the gitea server and locally.
+        """
+        def __init__(self, name):
+          self.name = name
+          self.path = "/tmp/repos/" + name
+          self.remote = "http://gitea:3000/test/" + name
+          self.remote_ssh = "ssh://gitea/root/" + name
+          self.git = f"git -C {self.path}"
+          self.create()
+
+        def create(self):
+          # create ssh remote repo
+          gitea.succeed(f"""
+            git init --bare -b main /root/{self.name}
+          """)
+          # create http remote repo
+          gitea.succeed(f"""
+            curl --fail -X POST http://{gitea_admin}:{gitea_admin_password}@gitea:3000/api/v1/user/repos \
+              -H 'Accept: application/json' -H 'Content-Type: application/json' \
+              -d {shlex.quote( f'{{"name":"{self.name}", "default_branch": "main"}}' )}
+          """)
+          # setup git remotes on client
+          client.succeed(f"""
+            mkdir -p {self.path} \
+            && git init -b main {self.path} \
+            && {self.git} remote add origin {self.remote} \
+            && {self.git} remote add origin-ssh root@gitea:{self.name}
+          """)
+    '';
+  };
+}
\ No newline at end of file
diff --git a/tests/nixos/fetch-git/testsupport/gitea.nix b/tests/nixos/fetch-git/testsupport/gitea.nix
index 2ea23961e..cf87bb466 100644
--- a/tests/nixos/fetch-git/testsupport/gitea.nix
+++ b/tests/nixos/fetch-git/testsupport/gitea.nix
@@ -15,6 +15,7 @@
 in {
   imports = [
     ../testsupport/setup.nix
+    ../testsupport/gitea-repo.nix
   ];
   nodes = {
     gitea = { pkgs, ... }: {
@@ -96,5 +97,6 @@ in {
     client.succeed("""
       ssh root@gitea true
     """)
+
   '';
 }
diff --git a/tests/nixos/fetch-git/testsupport/setup.nix b/tests/nixos/fetch-git/testsupport/setup.nix
index 8fc8e2e7c..a81d5614b 100644
--- a/tests/nixos/fetch-git/testsupport/setup.nix
+++ b/tests/nixos/fetch-git/testsupport/setup.nix
@@ -12,7 +12,10 @@ let
     ### TEST ${testCase.name}: ${testCase.description} ###
 
     with subtest("${testCase.description}"):
-        repo = Repo("${testCase.name}")
+        # Setup
+        ${indent testCase.setupScript}
+
+        # Test
         ${indent testCase.script}
   '';
 in
@@ -47,12 +50,19 @@ in
             A description of the test case.
           '';
         };
+        options.setupScript = mkOption {
+          type = types.lines;
+          description = ''
+            Python code that runs before the test case.
+          '';
+          default = "";
+        };
         options.script = mkOption {
           type = types.lines;
           description = ''
             Python code that runs the test.
 
-            Variables defined by `setupScript` will be available here.
+            Variables defined by the global `setupScript`, as well as `testCases.*.setupScript` will be available here.
           '';
         };
       });
@@ -67,36 +77,6 @@ in
       nix.settings.experimental-features = ["nix-command" "flakes"];
     };
     setupScript = ''
-      class Repo:
-        """
-        A class to create a git repository on the gitea server and locally.
-        """
-        def __init__(self, name):
-          self.name = name
-          self.path = "/tmp/repos/" + name
-          self.remote = "http://gitea:3000/test/" + name
-          self.remote_ssh = "ssh://gitea/root/" + name
-          self.git = f"git -C {self.path}"
-          self.create()
-
-        def create(self):
-          # create ssh remote repo
-          gitea.succeed(f"""
-            git init --bare -b main /root/{self.name}
-          """)
-          # create http remote repo
-          gitea.succeed(f"""
-            curl --fail -X POST http://{gitea_admin}:{gitea_admin_password}@gitea:3000/api/v1/user/repos \
-              -H 'Accept: application/json' -H 'Content-Type: application/json' \
-              -d {shlex.quote( f'{{"name":"{self.name}", "default_branch": "main"}}' )}
-          """)
-          # setup git remotes on client
-          client.succeed(f"""
-            mkdir -p {self.path} \
-            && git init -b main {self.path} \
-            && {self.git} remote add origin {self.remote} \
-            && {self.git} remote add origin-ssh root@gitea:{self.name}
-          """)
     '';
     testScript = ''
       start_all();

From 94eba0ebbbadfa48a4c1253cb94070a41310fae2 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 18 Jan 2024 13:42:41 +0100
Subject: [PATCH 405/421] tests/nixos/fetch-git: Memoize -> save

Memoization is for thunk-like behavior whereas this is executed eagerly.
---
 tests/nixos/fetch-git/test-cases/http-simple/default.nix | 2 +-
 tests/nixos/fetch-git/test-cases/ssh-simple/default.nix  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/nixos/fetch-git/test-cases/http-simple/default.nix b/tests/nixos/fetch-git/test-cases/http-simple/default.nix
index 1bd5bbba2..333ee45fd 100644
--- a/tests/nixos/fetch-git/test-cases/http-simple/default.nix
+++ b/tests/nixos/fetch-git/test-cases/http-simple/default.nix
@@ -8,7 +8,7 @@
       && {repo.git} commit -m 'commit1'
     """)
 
-    # memoize the revision
+    # save the revision
     rev1 = client.succeed(f"""
       {repo.git} rev-parse HEAD
     """).strip()
diff --git a/tests/nixos/fetch-git/test-cases/ssh-simple/default.nix b/tests/nixos/fetch-git/test-cases/ssh-simple/default.nix
index 0e4494ae0..f2deca141 100644
--- a/tests/nixos/fetch-git/test-cases/ssh-simple/default.nix
+++ b/tests/nixos/fetch-git/test-cases/ssh-simple/default.nix
@@ -8,7 +8,7 @@
       && {repo.git} commit -m 'commit1'
     """)
 
-    # memoize the revision
+    # save the revision
     rev1 = client.succeed(f"""
       {repo.git} rev-parse HEAD
     """).strip()

From 12541704052849d4160a13f7bbd873b40f19a3f9 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 18 Jan 2024 20:56:33 +0100
Subject: [PATCH 406/421] tests/nixos/fetch-git: Make the store paths unique

---
 tests/nixos/fetch-git/test-cases/http-simple/default.nix | 6 ++++--
 tests/nixos/fetch-git/test-cases/ssh-simple/default.nix  | 6 ++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/tests/nixos/fetch-git/test-cases/http-simple/default.nix b/tests/nixos/fetch-git/test-cases/http-simple/default.nix
index 333ee45fd..dcab8067e 100644
--- a/tests/nixos/fetch-git/test-cases/http-simple/default.nix
+++ b/tests/nixos/fetch-git/test-cases/http-simple/default.nix
@@ -1,10 +1,12 @@
+{ config, ... }:
 {
   description = "can fetch a git repo via http";
   script = ''
     # add a file to the repo
     client.succeed(f"""
-      echo chiang-mai > {repo.path}/thailand \
-      && {repo.git} add thailand \
+      echo ${config.name /* to make the git tree and store path unique */} > {repo.path}/test-case \
+      && echo chiang-mai > {repo.path}/thailand \
+      && {repo.git} add test-case thailand \
       && {repo.git} commit -m 'commit1'
     """)
 
diff --git a/tests/nixos/fetch-git/test-cases/ssh-simple/default.nix b/tests/nixos/fetch-git/test-cases/ssh-simple/default.nix
index f2deca141..f5fba1698 100644
--- a/tests/nixos/fetch-git/test-cases/ssh-simple/default.nix
+++ b/tests/nixos/fetch-git/test-cases/ssh-simple/default.nix
@@ -1,10 +1,12 @@
+{ config, ... }:
 {
   description = "can fetch a git repo via ssh";
   script = ''
     # add a file to the repo
     client.succeed(f"""
-      echo chiang-mai > {repo.path}/thailand \
-      && {repo.git} add thailand \
+      echo ${config.name /* to make the git tree and store path unique */} > {repo.path}/test-case \
+      && echo chiang-mai > {repo.path}/thailand \
+      && {repo.git} add test-case thailand \
       && {repo.git} commit -m 'commit1'
     """)
 

From ed975e953c30c335f8403352acc785323a5a925c Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 18 Jan 2024 20:59:24 +0100
Subject: [PATCH 407/421] tests/nixos/fetch-git: Testsupport for private repos

---
 .../fetch-git/testsupport/gitea-repo.nix      | 36 +++++++++++++++----
 1 file changed, 30 insertions(+), 6 deletions(-)

diff --git a/tests/nixos/fetch-git/testsupport/gitea-repo.nix b/tests/nixos/fetch-git/testsupport/gitea-repo.nix
index 916552bb2..a3ad65ca4 100644
--- a/tests/nixos/fetch-git/testsupport/gitea-repo.nix
+++ b/tests/nixos/fetch-git/testsupport/gitea-repo.nix
@@ -1,11 +1,31 @@
 { lib, ... }:
 let
-  inherit (lib) mkOption types;
+  inherit (lib)
+    mkIf
+    mkOption
+    types
+    ;
+
+  boolPyLiteral = b: if b then "True" else "False";
 
   testCaseExtension = { config, ... }: {
-    setupScript = ''
-      repo = Repo("${config.name}")
-    '';
+    options = {
+      repo.enable = mkOption {
+        type = types.bool;
+        default = true;
+        description = "Whether to provide a repo variable - automatic repo creation.";
+      };
+      repo.private = mkOption {
+        type = types.bool;
+        default = false;
+        description = "Whether the repo should be private.";
+      };
+    };
+    config = mkIf config.repo.enable {
+      setupScript = ''
+        repo = Repo("${config.name}", private=${boolPyLiteral config.repo.private})
+      '';
+    };
   };
 in
 {
@@ -16,16 +36,20 @@ in
   };
   config = {
     setupScript = ''
+      def boolToJSON(b):
+        return "true" if b else "false"
+
       class Repo:
         """
         A class to create a git repository on the gitea server and locally.
         """
-        def __init__(self, name):
+        def __init__(self, name, private=False):
           self.name = name
           self.path = "/tmp/repos/" + name
           self.remote = "http://gitea:3000/test/" + name
           self.remote_ssh = "ssh://gitea/root/" + name
           self.git = f"git -C {self.path}"
+          self.private = private
           self.create()
 
         def create(self):
@@ -37,7 +61,7 @@ in
           gitea.succeed(f"""
             curl --fail -X POST http://{gitea_admin}:{gitea_admin_password}@gitea:3000/api/v1/user/repos \
               -H 'Accept: application/json' -H 'Content-Type: application/json' \
-              -d {shlex.quote( f'{{"name":"{self.name}", "default_branch": "main"}}' )}
+              -d {shlex.quote( f'{{"name":"{self.name}", "default_branch": "main", "private": {boolToJSON(self.private)}}}' )}
           """)
           # setup git remotes on client
           client.succeed(f"""

From 76a50b3a69dd7202fa4c68ca8d12fde152e6341a Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 18 Jan 2024 22:25:30 +0100
Subject: [PATCH 408/421] doc: GitRepoImpl::path

---
 src/libfetchers/git-utils.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 6726407b5..f34329fab 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -139,6 +139,7 @@ T peelObject(git_repository * repo, git_object * obj, git_object_t type)
 
 struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 {
+    /** Location of the repository on disk. */
     CanonPath path;
     Repository repo;
 

From 8d422c2fef4309b4b7de8e2f909957775a9ec3ef Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 18 Jan 2024 22:26:24 +0100
Subject: [PATCH 409/421] Revert libgit2 fetching

libgit2 is not capable of using git-credentials helpers yet.
This prevents private repositories from being used.

Based on code that was replaced in https://github.com/NixOS/nix/pull/9240
(Introduce libgit2); hence:

Co-authored-by: Eelco Dolstra <edolstra@gmail.com>
---
 src/libfetchers/git-utils.cc | 31 ++++++++++++-------------------
 1 file changed, 12 insertions(+), 19 deletions(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index f34329fab..911c16c4b 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -383,27 +383,20 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
     {
         Activity act(*logger, lvlTalkative, actFetchTree, fmt("fetching Git repository '%s'", url));
 
-        Remote remote;
+        // TODO: implement git-credential helper support (preferably via libgit2, which as of 2024-01 does not support that)
+        //       then use code that was removed in this commit (see blame)
 
-        if (git_remote_create_anonymous(Setter(remote), *this, url.c_str()))
-            throw Error("cannot create Git remote '%s': %s", url, git_error_last()->message);
+        auto dir = this->path;
 
-        char * refspecs[] = {(char *) refspec.c_str()};
-        git_strarray refspecs2 {
-            .strings = refspecs,
-            .count = 1
-        };
-
-        git_fetch_options opts = GIT_FETCH_OPTIONS_INIT;
-        // FIXME: for some reason, shallow fetching over ssh barfs
-        // with "could not read from remote repository".
-        opts.depth = shallow && parseURL(url).scheme != "ssh" ? 1 : GIT_FETCH_DEPTH_FULL;
-        opts.callbacks.payload = &act;
-        opts.callbacks.sideband_progress = sidebandProgressCallback;
-        opts.callbacks.transfer_progress = transferProgressCallback;
-
-        if (git_remote_fetch(remote.get(), &refspecs2, &opts, nullptr))
-            throw Error("fetching '%s' from '%s': %s", refspec, url, git_error_last()->message);
+        runProgram(RunOptions {
+            .program = "git",
+            .searchPath = true,
+            // FIXME: git stderr messes up our progress indicator, so
+            // we're using --quiet for now. Should process its stderr.
+            .args = { "-C", path.abs(), "fetch", "--quiet", "--force", "--", url, refspec },
+            .input = {},
+            .isInteractive = true
+        });
     }
 
     void verifyCommit(

From 346d513d86491f2040735d22ba49cb0d701edb70 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Thu, 18 Jan 2024 22:34:38 +0100
Subject: [PATCH 410/421] tests/nixos/fetch-git: Add http-auth test

---
 .../test-cases/http-auth/default.nix          | 40 +++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 tests/nixos/fetch-git/test-cases/http-auth/default.nix

diff --git a/tests/nixos/fetch-git/test-cases/http-auth/default.nix b/tests/nixos/fetch-git/test-cases/http-auth/default.nix
new file mode 100644
index 000000000..d483d54fb
--- /dev/null
+++ b/tests/nixos/fetch-git/test-cases/http-auth/default.nix
@@ -0,0 +1,40 @@
+{ config, ... }:
+{
+  description = "can fetch a private git repo via http";
+  repo.private = true;
+  script = ''
+    # add a file to the repo
+    client.succeed(f"""
+      echo ${config.name /* to make the git tree and store path unique */} > {repo.path}/test-case \
+      && echo lutyabrook > {repo.path}/new-york-state \
+      && {repo.git} add test-case new-york-state \
+      && {repo.git} commit -m 'commit1'
+    """)
+
+    # memoize the revision
+    rev1 = client.succeed(f"""
+      {repo.git} rev-parse HEAD
+    """).strip()
+
+    # push to the server
+    client.succeed(f"""
+      {repo.git} push origin main
+    """)
+
+    # fetch the repo via nix
+    fetched1 = client.succeed(f"""
+      nix eval --impure --raw --expr "(builtins.fetchGit {repo.remote}).outPath"
+    """)
+
+    # check if the committed file is there
+    client.succeed(f"""
+      test -f {fetched1}/new-york-state
+    """)
+
+    # check if the revision is the same
+    rev1_fetched = client.succeed(f"""
+      nix eval --impure --raw --expr "(builtins.fetchGit {repo.remote}).rev"
+    """).strip()
+    assert rev1 == rev1_fetched, f"rev1: {rev1} != rev1_fetched: {rev1_fetched}"
+  '';
+}

From 14f470ec4e9d481698b97ea2dae101693fbaca95 Mon Sep 17 00:00:00 2001
From: Robert Hensing <roberth@users.noreply.github.com>
Date: Fri, 19 Jan 2024 00:32:30 +0100
Subject: [PATCH 411/421] doc/hacking.md: Hint short option `make -e`

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 doc/manual/src/contributing/hacking.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md
index fe91787a3..9a7623dc9 100644
--- a/doc/manual/src/contributing/hacking.md
+++ b/doc/manual/src/contributing/hacking.md
@@ -115,7 +115,7 @@ You can also build Nix for one of the [supported platforms](#platforms).
 
 You may need `profiledir=$out/etc/profile.d` and `sysconfdir=$out/etc` to run `make install`.
 
-Run `make` with [`--environment-overrides`](https://www.gnu.org/software/make/manual/make.html#index-_002de) to allow environment variables to override `Makefile` variables:
+Run `make` with [`-e` / `--environment-overrides`](https://www.gnu.org/software/make/manual/make.html#index-_002de) to allow environment variables to override `Makefile` variables:
 
 - `ENABLE_BUILD=yes` to enable building the C++ code.
 - `ENABLE_DOC_GEN=yes` to enable building the documentation (manual, man pages, etc.).

From 28d7db249ace91c10a9ad6cb6d11a6c2109929fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?=
 <7226587+thufschmitt@users.noreply.github.com>
Date: Fri, 19 Jan 2024 10:10:00 +0100
Subject: [PATCH 412/421] Remove a nonsensical shorthand flag in `nix store
 add`

`-n` was an alias for `--mode`, but that seems to just be a copy-paste error as it doesn't make sense.
`--mode` probably doesn't need a shorthand flag at all, so remove it.

Noticed in https://github.com/NixOS/nix/pull/9809#issuecomment-1899890555
---
 src/nix/add-to-store.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index 64a43ecfa..171848002 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -38,7 +38,6 @@ struct CmdAddToStore : MixDryRun, StoreCommand
 
         addFlag({
             .longName  = "mode",
-            .shortName = 'n',
             .description = R"(
     How to compute the hash of the input.
     One of:

From bc00fa46472c56ccfddc2d6e81453be537d2e051 Mon Sep 17 00:00:00 2001
From: DavHau <hsngrmpf+github@gmail.com>
Date: Fri, 19 Jan 2024 15:59:15 +0700
Subject: [PATCH 413/421] fetchTree/fetchGit: re-enable shallow fetching

Add several tests for git fetching:
- shallow-cache-separation: can fetch the same repo shallowly and non-shallowly
- shallow-ignore-ref: ensure that ref gets ignored when shallow=true is set
- ssh-shallow: can fetch a git repo via ssh using shallow=1
---
 src/libfetchers/git-utils.cc                  |  9 ++-
 src/libfetchers/git.cc                        | 16 ++++--
 .../shallow-cache-separation/default.nix      | 57 +++++++++++++++++++
 .../test-cases/shallow-ignore-ref/default.nix | 40 +++++++++++++
 .../test-cases/ssh-shallow/default.nix        | 52 +++++++++++++++++
 .../fetch-git/testsupport/gitea-repo.nix      |  2 +-
 6 files changed, 168 insertions(+), 8 deletions(-)
 create mode 100644 tests/nixos/fetch-git/test-cases/shallow-cache-separation/default.nix
 create mode 100644 tests/nixos/fetch-git/test-cases/shallow-ignore-ref/default.nix
 create mode 100644 tests/nixos/fetch-git/test-cases/ssh-shallow/default.nix

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 911c16c4b..382a363f0 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -387,13 +387,20 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         //       then use code that was removed in this commit (see blame)
 
         auto dir = this->path;
+        Strings gitArgs;
+        if (shallow) {
+            gitArgs = { "-C", dir.abs(), "fetch", "--quiet", "--force", "--depth", "1", "--", url, refspec };
+        }
+        else {
+            gitArgs = { "-C", dir.abs(), "fetch", "--quiet", "--force", "--", url, refspec };
+        }
 
         runProgram(RunOptions {
             .program = "git",
             .searchPath = true,
             // FIXME: git stderr messes up our progress indicator, so
             // we're using --quiet for now. Should process its stderr.
-            .args = { "-C", path.abs(), "fetch", "--quiet", "--force", "--", url, refspec },
+            .args = gitArgs,
             .input = {},
             .isInteractive = true
         });
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 6ecb7a4ea..f9a1cb1bc 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -50,10 +50,12 @@ bool touchCacheFile(const Path & path, time_t touch_time)
     return lutimes(path.c_str(), times) == 0;
 }
 
-Path getCachePath(std::string_view key)
+Path getCachePath(std::string_view key, bool shallow)
 {
-    return getCacheDir() + "/nix/gitv3/" +
-        hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false);
+    return getCacheDir()
+    + "/nix/gitv3/"
+    + hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false)
+    + (shallow ? "-shallow" : "");
 }
 
 // Returns the name of the HEAD branch.
@@ -92,7 +94,8 @@ std::optional<std::string> readHead(const Path & path)
 // Persist the HEAD ref from the remote repo in the local cached repo.
 bool storeCachedHead(const std::string & actualUrl, const std::string & headRef)
 {
-    Path cacheDir = getCachePath(actualUrl);
+    // set shallow=false as HEAD will never be queried for a shallow repo
+    Path cacheDir = getCachePath(actualUrl, false);
     try {
         runProgram("git", true, { "-C", cacheDir, "--git-dir", ".", "symbolic-ref", "--", "HEAD", headRef });
     } catch (ExecError &e) {
@@ -107,7 +110,8 @@ std::optional<std::string> readHeadCached(const std::string & actualUrl)
 {
     // Create a cache path to store the branch of the HEAD ref. Append something
     // in front of the URL to prevent collision with the repository itself.
-    Path cacheDir = getCachePath(actualUrl);
+    // set shallow=false as HEAD will never be queried for a shallow repo
+    Path cacheDir = getCachePath(actualUrl, false);
     Path headRefFile = cacheDir + "/HEAD";
 
     time_t now = time(0);
@@ -508,7 +512,7 @@ struct GitInputScheme : InputScheme
             if (!input.getRev())
                 input.attrs.insert_or_assign("rev", GitRepo::openRepo(CanonPath(repoDir))->resolveRef(ref).gitRev());
         } else {
-            Path cacheDir = getCachePath(repoInfo.url);
+            Path cacheDir = getCachePath(repoInfo.url, getShallowAttr(input));
             repoDir = cacheDir;
             repoInfo.gitDir = ".";
 
diff --git a/tests/nixos/fetch-git/test-cases/shallow-cache-separation/default.nix b/tests/nixos/fetch-git/test-cases/shallow-cache-separation/default.nix
new file mode 100644
index 000000000..57561e74b
--- /dev/null
+++ b/tests/nixos/fetch-git/test-cases/shallow-cache-separation/default.nix
@@ -0,0 +1,57 @@
+{
+  description = "can fetch the same repo shallowly and non-shallowly";
+  script = ''
+    # create branch1 off of main
+    client.succeed(f"""
+      echo chiang-mai > {repo.path}/thailand \
+      && {repo.git} add thailand \
+      && {repo.git} commit -m 'commit1' \
+      \
+      && {repo.git} push origin --all
+    """)
+
+    # save the revision
+    mainRev = client.succeed(f"""
+      {repo.git} rev-parse main
+    """).strip()
+
+    # fetch shallowly
+    revCountShallow = client.succeed(f"""
+      nix eval --impure --expr '
+        (builtins.fetchGit {{
+          url = "{repo.remote}";
+          rev = "{mainRev}";
+          shallow = true;
+        }}).revCount
+      '
+    """).strip()
+    # ensure the revCount is 0
+    assert revCountShallow == "0", f"revCountShallow should be 0, but is {revCountShallow}"
+
+    # fetch non-shallowly
+    revCountNonShallow = client.succeed(f"""
+      nix eval --impure --expr '
+        (builtins.fetchGit {{
+          url = "{repo.remote}";
+          rev = "{mainRev}";
+          shallow = false;
+        }}).revCount
+      '
+    """).strip()
+    # ensure the revCount is 1
+    assert revCountNonShallow == "1", f"revCountNonShallow should be 1, but is {revCountNonShallow}"
+
+    # fetch shallowly again
+    revCountShallow2 = client.succeed(f"""
+      nix eval --impure --expr '
+        (builtins.fetchGit {{
+          url = "{repo.remote}";
+          rev = "{mainRev}";
+          shallow = true;
+        }}).revCount
+      '
+    """).strip()
+    # ensure the revCount is 0
+    assert revCountShallow2 == "0", f"revCountShallow2 should be 0, but is {revCountShallow2}"
+  '';
+}
diff --git a/tests/nixos/fetch-git/test-cases/shallow-ignore-ref/default.nix b/tests/nixos/fetch-git/test-cases/shallow-ignore-ref/default.nix
new file mode 100644
index 000000000..456ee8341
--- /dev/null
+++ b/tests/nixos/fetch-git/test-cases/shallow-ignore-ref/default.nix
@@ -0,0 +1,40 @@
+{
+  description = "ensure that ref gets ignored when shallow=true is set";
+  script = ''
+    # create branch1 off of main
+    client.succeed(f"""
+      echo chiang-mai > {repo.path}/thailand \
+      && {repo.git} add thailand \
+      && {repo.git} commit -m 'commit1' \
+      \
+      && {repo.git} checkout -b branch1 main \
+      && echo bangkok > {repo.path}/thailand \
+      && {repo.git} add thailand \
+      && {repo.git} commit -m 'commit2' \
+      \
+      && {repo.git} push origin --all
+    """)
+
+    # save the revisions
+    mainRev = client.succeed(f"""
+      {repo.git} rev-parse main
+    """).strip()
+    branch1Rev = client.succeed(f"""
+      {repo.git} rev-parse branch1
+    """).strip()
+
+    # Ensure that ref gets ignored when fetching shallowly.
+    # This would fail if the ref was respected, as branch1Rev is not on main.
+    client.succeed(f"""
+      nix eval --impure --raw --expr '
+        (builtins.fetchGit {{
+          url = "{repo.remote}";
+          rev = "{branch1Rev}";
+          ref = "main";
+          shallow = true;
+        }})
+      '
+    """)
+
+  '';
+}
diff --git a/tests/nixos/fetch-git/test-cases/ssh-shallow/default.nix b/tests/nixos/fetch-git/test-cases/ssh-shallow/default.nix
new file mode 100644
index 000000000..979512af9
--- /dev/null
+++ b/tests/nixos/fetch-git/test-cases/ssh-shallow/default.nix
@@ -0,0 +1,52 @@
+{
+  description = "can fetch a git repo via ssh using shallow=1";
+  script = ''
+    # add a file to the repo
+    client.succeed(f"""
+      echo chiang-mai > {repo.path}/thailand \
+      && {repo.git} add thailand \
+      && {repo.git} commit -m 'commit1'
+    """)
+
+    # memoize the revision
+    rev1 = client.succeed(f"""
+      {repo.git} rev-parse HEAD
+    """).strip()
+
+    # push to the server
+    client.succeed(f"""
+      {repo.git} push origin-ssh main
+    """)
+
+    fetchGit_expr = f"""
+      builtins.fetchGit {{
+        url = "{repo.remote_ssh}";
+        rev = "{rev1}";
+        shallow = true;
+      }}
+    """
+
+    # fetch the repo via nix
+    fetched1 = client.succeed(f"""
+      nix eval --impure --raw --expr '({fetchGit_expr}).outPath'
+    """)
+
+    # check if the committed file is there
+    client.succeed(f"""
+      test -f {fetched1}/thailand
+    """)
+
+    # check if the revision is the same
+    rev1_fetched = client.succeed(f"""
+      nix eval --impure --raw --expr '({fetchGit_expr}).rev'
+    """).strip()
+    assert rev1 == rev1_fetched, f"rev1: {rev1} != rev1_fetched: {rev1_fetched}"
+
+    # check if revCount is 1
+    revCount1 = client.succeed(f"""
+      nix eval --impure --expr '({fetchGit_expr}).revCount'
+    """).strip()
+    print(f"revCount1: {revCount1}")
+    assert revCount1 == '0', f"rev count is not 0 but {revCount1}"
+  '';
+}
diff --git a/tests/nixos/fetch-git/testsupport/gitea-repo.nix b/tests/nixos/fetch-git/testsupport/gitea-repo.nix
index a3ad65ca4..e9f4adcc1 100644
--- a/tests/nixos/fetch-git/testsupport/gitea-repo.nix
+++ b/tests/nixos/fetch-git/testsupport/gitea-repo.nix
@@ -72,4 +72,4 @@ in
           """)
     '';
   };
-}
\ No newline at end of file
+}

From 75a6e6dd0eb60f3bcaaa3b33b085fb542638eb44 Mon Sep 17 00:00:00 2001
From: Yuxuan Shui <yshuiv7@gmail.com>
Date: Thu, 18 Jan 2024 16:39:34 +0000
Subject: [PATCH 414/421] Add --unpack to nix store prefetch-file

---
 src/nix/prefetch.cc | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc
index b5d619006..84b79ea28 100644
--- a/src/nix/prefetch.cc
+++ b/src/nix/prefetch.cc
@@ -262,6 +262,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON
 {
     std::string url;
     bool executable = false;
+    bool unpack = false;
     std::optional<std::string> name;
     HashAlgorithm hashAlgo = HashAlgorithm::SHA256;
     std::optional<Hash> expectedHash;
@@ -294,6 +295,14 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON
             .handler = {&executable, true},
         });
 
+        addFlag({
+            .longName = "unpack",
+            .description =
+                "Unpack the archive (which must be a tarball or zip file) and add "
+                "the result to the Nix store.",
+            .handler = {&unpack, true},
+        });
+
         expectArg("url", &url);
     }
 
@@ -310,7 +319,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON
     }
     void run(ref<Store> store) override
     {
-        auto [storePath, hash] = prefetchFile(store, url, name, hashAlgo, expectedHash, false, executable);
+        auto [storePath, hash] = prefetchFile(store, url, name, hashAlgo, expectedHash, unpack, executable);
 
         if (json) {
             auto res = nlohmann::json::object();

From 8983ee8b2e0c10e6cac672a5a7ada4698235a62e Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Wed, 17 Jan 2024 17:54:03 +0100
Subject: [PATCH 415/421] refactor: Un-callback transform
 Store::queryPathInfoFromClientCache

This part of the code was not necessarily callback based.
Removing CPS is always nice; particularly if there's no loss of
functionality, like here.
---
 src/libstore/store-api.cc | 18 +++++++++---------
 src/libstore/store-api.hh |  8 ++++----
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 66bc95625..f237578e5 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -685,8 +685,7 @@ static bool goodStorePath(const StorePath & expected, const StorePath & actual)
         && (expected.name() == Store::MissingName || expected.name() == actual.name());
 }
 
-bool Store::queryPathInfoFromClientCache(const StorePath & storePath,
-    Callback<ref<const ValidPathInfo>> & callback)
+std::optional<ref<const ValidPathInfo>> Store::queryPathInfoFromClientCache(const StorePath & storePath)
 {
     auto hashPart = std::string(storePath.hashPart());
 
@@ -696,8 +695,7 @@ bool Store::queryPathInfoFromClientCache(const StorePath & storePath,
             stats.narInfoReadAverted++;
             if (!res->didExist())
                 throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
-            callback(ref<const ValidPathInfo>(res->value));
-            return true;
+            return ref(res->value);
         }
     }
 
@@ -713,12 +711,11 @@ bool Store::queryPathInfoFromClientCache(const StorePath & storePath,
                     !goodStorePath(storePath, res.second->path))
                     throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
             }
-            callback(ref<const ValidPathInfo>(res.second));
-            return true;
+            return ref(res.second);
         }
     }
 
-    return false;
+    return std::nullopt;
 }
 
 
@@ -728,8 +725,11 @@ void Store::queryPathInfo(const StorePath & storePath,
     auto hashPart = std::string(storePath.hashPart());
 
     try {
-        if (queryPathInfoFromClientCache(storePath, callback))
-            return;
+        auto r = queryPathInfoFromClientCache(storePath);
+        if (r.has_value()) {
+            ref<const ValidPathInfo> & info = *r;
+            return callback(ref(info));
+        }
     } catch (...) { return callback.rethrow(); }
 
     auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 2a1092d9e..e47f2c768 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -285,14 +285,14 @@ public:
     /**
      * NOTE: this is not the final interface - to be modified in next commit.
      *
-     * Asynchronous version that only queries the local narinfo cache and not
+     * Version of queryPathInfo() that only queries the local narinfo cache and not
      * the actual store.
      *
-     * @return true if the path was known and the callback invoked
-     * @return false if the path was not known and the callback not invoked
+     * @return `std::make_optional(vpi)` if the path is known
+     * @return `std::null_opt` if the path was not known to be valid or invalid
      * @throw InvalidPathError if the path is known to be invalid
      */
-    bool queryPathInfoFromClientCache(const StorePath & path, Callback<ref<const ValidPathInfo>> & callback);
+    std::optional<ref<const ValidPathInfo>> queryPathInfoFromClientCache(const StorePath & path);
 
     /**
      * Query the information about a realisation.

From d19627e8b4c3c09b0cc1329a9acaa8e5b070f26e Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Fri, 19 Jan 2024 17:00:39 +0100
Subject: [PATCH 416/421] refactor: Remove throw from
 queryPathInfoFromClientCache

Return a value instead of throwing.

Rather than the more trivial refactor of wrapping the return value in
another std::optional, we retain the meaning of the outer optional:
"we know at least something."

So we have changed:
return nullopt    -> return nullopt
throw InvalidPath -> return make_optional(nullptr)
return vpi        -> return make_optional(vpi)
---
 src/libstore/store-api.cc | 22 ++++++++++++++--------
 src/libstore/store-api.hh | 10 ++++------
 2 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index f237578e5..2cd40d510 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -685,7 +685,8 @@ static bool goodStorePath(const StorePath & expected, const StorePath & actual)
         && (expected.name() == Store::MissingName || expected.name() == actual.name());
 }
 
-std::optional<ref<const ValidPathInfo>> Store::queryPathInfoFromClientCache(const StorePath & storePath)
+
+std::optional<std::shared_ptr<const ValidPathInfo>> Store::queryPathInfoFromClientCache(const StorePath & storePath)
 {
     auto hashPart = std::string(storePath.hashPart());
 
@@ -693,9 +694,10 @@ std::optional<ref<const ValidPathInfo>> Store::queryPathInfoFromClientCache(cons
         auto res = state.lock()->pathInfoCache.get(std::string(storePath.to_string()));
         if (res && res->isKnownNow()) {
             stats.narInfoReadAverted++;
-            if (!res->didExist())
-                throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
-            return ref(res->value);
+            if (res->didExist())
+                return std::make_optional(res->value);
+            else
+                return std::make_optional(nullptr);
         }
     }
 
@@ -709,9 +711,10 @@ std::optional<ref<const ValidPathInfo>> Store::queryPathInfoFromClientCache(cons
                     res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
                 if (res.first == NarInfoDiskCache::oInvalid ||
                     !goodStorePath(storePath, res.second->path))
-                    throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
+                    return std::make_optional(nullptr);
             }
-            return ref(res.second);
+            assert(res.second);
+            return std::make_optional(res.second);
         }
     }
 
@@ -727,8 +730,11 @@ void Store::queryPathInfo(const StorePath & storePath,
     try {
         auto r = queryPathInfoFromClientCache(storePath);
         if (r.has_value()) {
-            ref<const ValidPathInfo> & info = *r;
-            return callback(ref(info));
+            std::shared_ptr<const ValidPathInfo> & info = *r;
+            if (info)
+                return callback(ref(info));
+            else
+                throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
         }
     } catch (...) { return callback.rethrow(); }
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index e47f2c768..2f8a9440e 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -283,16 +283,14 @@ public:
         Callback<ref<const ValidPathInfo>> callback) noexcept;
 
     /**
-     * NOTE: this is not the final interface - to be modified in next commit.
-     *
      * Version of queryPathInfo() that only queries the local narinfo cache and not
      * the actual store.
      *
-     * @return `std::make_optional(vpi)` if the path is known
-     * @return `std::null_opt` if the path was not known to be valid or invalid
-     * @throw InvalidPathError if the path is known to be invalid
+     * @return `std::nullopt` if nothing is known about the path in the local narinfo cache.
+     * @return `std::make_optional(nullptr)` if the path is known to not exist.
+     * @return `std::make_optional(validPathInfo)` if the path is known to exist.
      */
-    std::optional<ref<const ValidPathInfo>> queryPathInfoFromClientCache(const StorePath & path);
+    std::optional<std::shared_ptr<const ValidPathInfo>> queryPathInfoFromClientCache(const StorePath & path);
 
     /**
      * Query the information about a realisation.

From 356352c3709f69b6d11ed7f14ffa586219170908 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 19 Jan 2024 01:07:26 -0500
Subject: [PATCH 417/421] Add missing `--hash-algo` flag to `nix store add`

---
 doc/manual/rl-next/nix-store-add.md | 7 +++++++
 src/nix/add-to-store.cc             | 7 +++++--
 tests/functional/add.sh             | 2 ++
 3 files changed, 14 insertions(+), 2 deletions(-)
 create mode 100644 doc/manual/rl-next/nix-store-add.md

diff --git a/doc/manual/rl-next/nix-store-add.md b/doc/manual/rl-next/nix-store-add.md
new file mode 100644
index 000000000..d55711569
--- /dev/null
+++ b/doc/manual/rl-next/nix-store-add.md
@@ -0,0 +1,7 @@
+---
+synopsis: Give `nix store add` a `--hash-algo` flag
+prs: 9809
+---
+
+Adds a missing feature that was present in the old CLI, and matches our
+plans to have similar flags for `nix hash convert` and `hash hash path`.
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index 171848002..f2dbe8a2c 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -22,6 +22,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
     Path path;
     std::optional<std::string> namePart;
     ContentAddressMethod caMethod = FileIngestionMethod::Recursive;
+    HashAlgorithm hashAlgo = HashAlgorithm::SHA256;
 
     CmdAddToStore()
     {
@@ -51,6 +52,8 @@ struct CmdAddToStore : MixDryRun, StoreCommand
                 this->caMethod = parseIngestionMethod(s);
             }},
         });
+
+        addFlag(Flag::mkHashAlgoFlag("hash-algo", &hashAlgo));
     }
 
     void run(ref<Store> store) override
@@ -63,9 +66,9 @@ struct CmdAddToStore : MixDryRun, StoreCommand
 
         auto storePath = dryRun
             ? store->computeStorePath(
-                *namePart, accessor, path2, caMethod, HashAlgorithm::SHA256, {}).first
+                *namePart, accessor, path2, caMethod, hashAlgo, {}).first
             : store->addToStoreSlow(
-                *namePart, accessor, path2, caMethod, HashAlgorithm::SHA256, {}).path;
+                *namePart, accessor, path2, caMethod, hashAlgo, {}).path;
 
         logger->cout("%s", store->printStorePath(storePath));
     }
diff --git a/tests/functional/add.sh b/tests/functional/add.sh
index d0fedcb25..762e01dbe 100644
--- a/tests/functional/add.sh
+++ b/tests/functional/add.sh
@@ -37,9 +37,11 @@ clearStore
     path3=$(nix store add-path ./dummy)
     [[ "$path1" == "$path2" ]]
     [[ "$path1" == "$path3" ]]
+    path4=$(nix store add --mode nar --hash-algo sha1 ./dummy)
 )
 (
     path1=$(nix store add --mode flat ./dummy)
     path2=$(nix store add-file ./dummy)
     [[ "$path1" == "$path2" ]]
+    path4=$(nix store add --mode flat --hash-algo sha1 ./dummy)
 )

From edf3ecc497d9931f84d8a28679b51773c761fdd8 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 22 Oct 2023 20:01:01 -0400
Subject: [PATCH 418/421] Document JSON formats

Good to document these formats separately from commands that happen to
use them.

Eventually I would like this and `builtins.derivation` to refer to a
store section on derivations that is authoritative, but that doesn't yet
exist, and will take some time to make. So I think we're just best off
merging this now as is.

Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
---
 doc/manual/src/SUMMARY.md.in             |  3 +
 doc/manual/src/glossary.md               |  2 +-
 doc/manual/src/json/derivation.md        | 71 +++++++++++++++++
 doc/manual/src/json/store-object-info.md | 97 ++++++++++++++++++++++++
 src/libstore/globals.hh                  |  2 +-
 src/nix/derivation-add.md                |  7 +-
 src/nix/derivation-show.md               | 60 +--------------
 7 files changed, 181 insertions(+), 61 deletions(-)
 create mode 100644 doc/manual/src/json/derivation.md
 create mode 100644 doc/manual/src/json/store-object-info.md

diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in
index c67ddc6cb..10fe51fc9 100644
--- a/doc/manual/src/SUMMARY.md.in
+++ b/doc/manual/src/SUMMARY.md.in
@@ -104,6 +104,9 @@
     - [Channels](command-ref/files/channels.md)
     - [Default Nix expression](command-ref/files/default-nix-expression.md)
 - [Architecture and Design](architecture/architecture.md)
+- [JSON Formats](json/index.md)
+  - [Store Object Info](json/store-object-info.md)
+  - [Derivation](json/derivation.md)
 - [Protocols](protocols/index.md)
   - [Serving Tarball Flakes](protocols/tarball-fetcher.md)
   - [Derivation "ATerm" file format](protocols/derivation-aterm.md)
diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md
index 3c0570a44..124dc8d2e 100644
--- a/doc/manual/src/glossary.md
+++ b/doc/manual/src/glossary.md
@@ -127,7 +127,7 @@
   non-[fixed-output](#gloss-fixed-output-derivation)
   derivation.
 
-- [output-addressed store object]{#gloss-output-addressed-store-object}
+- [content-addressed store object]{#gloss-content-addressed-store-object}
 
   A [store object] whose [store path] is determined by its contents.
   This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation).
diff --git a/doc/manual/src/json/derivation.md b/doc/manual/src/json/derivation.md
new file mode 100644
index 000000000..649d543cc
--- /dev/null
+++ b/doc/manual/src/json/derivation.md
@@ -0,0 +1,71 @@
+# Derivation JSON Format
+
+> **Warning**
+>
+> This JSON format is currently
+> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command)
+> and subject to change.
+
+The JSON serialization of a
+[derivations](@docroot@/glossary.md#gloss-store-derivation)
+is a JSON object with the following fields:
+
+* `name`:
+  The name of the derivation.
+  This is used when calculating the store paths of the derivation's outputs.
+
+* `outputs`:
+  Information about the output paths of the derivation.
+  This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields:
+
+  * `path`: The output path.
+
+  * `hashAlgo`:
+    For fixed-output derivations, the hashing algorithm (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a NAR hash rather than a flat file hash.
+
+  * `hash`:
+    For fixed-output derivations, the expected content hash in base-16.
+
+  > **Example**
+  >
+  > ```json
+  > "outputs": {
+  >   "out": {
+  >     "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source",
+  >     "hashAlgo": "r:sha256",
+  >     "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62"
+  >   }
+  > }
+  > ```
+
+* `inputSrcs`:
+  A list of store paths on which this derivation depends.
+
+* `inputDrvs`:
+  A JSON object specifying the derivations on which this derivation depends, and what outputs of those derivations.
+
+  > **Example**
+  >
+  > ```json
+  > "inputDrvs": {
+  >   "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"],
+  >   "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"]
+  > }
+  > ```
+
+  specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`.
+
+* `system`:
+  The system type on which this derivation is to be built
+  (e.g. `x86_64-linux`).
+
+* `builder`:
+  The absolute path of the program to be executed to run the build.
+  Typically this is the `bash` shell
+  (e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`).
+
+* `args`:
+  The command-line arguments passed to the `builder`.
+
+* `env`:
+  The environment passed to the `builder`.
diff --git a/doc/manual/src/json/store-object-info.md b/doc/manual/src/json/store-object-info.md
new file mode 100644
index 000000000..db43c2fa1
--- /dev/null
+++ b/doc/manual/src/json/store-object-info.md
@@ -0,0 +1,97 @@
+# Store object info JSON format
+
+> **Warning**
+>
+> This JSON format is currently
+> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command)
+> and subject to change.
+
+Info about a [store object].
+
+* `path`:
+
+  [Store path][store path] to the given store object.
+
+* `narHash`:
+
+  Hash of the [file system object] part of the store object when serialized as a [Nix Archive](#gloss-nar).
+
+* `narSize`:
+
+  Size of the [file system object] part of the store object when serialized as a [Nix Archive](#gloss-nar).
+
+* `references`:
+
+  An array of [store paths][store path], possibly including this one.
+
+* `ca` (optional):
+
+  Content address of this store object's file system object, used to compute its store path.
+
+[store path]: @docroot@/glossary.md#gloss-store-path
+[file system object]: @docroot@/store/file-system-object.md
+
+## Impure fields
+
+These are not intrinsic properties of the store object.
+In other words, the same store object residing in different store could have different values for these properties.
+
+* `deriver` (optional):
+
+  The path to the [derivation] from which this store object is produced.
+
+  [derivation]: @docroot@/glossary.md#gloss-store-derivation
+
+* `registrationTime` (optional):
+
+  When this derivation was added to the store.
+
+* `ultimate` (optional):
+
+  Whether this store object is trusted because we built it ourselves, rather than substituted a build product from elsewhere.
+
+* `signatures` (optional):
+
+  Signatures claiming that this store object is what it claims to be.
+  Not relevant for [content-addressed] store objects,
+  but useful for [input-addressed] store objects.
+
+  [content-addressed]: @docroot@/glossary.md#gloss-content-addressed-store-object
+  [input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object
+
+### `.narinfo` extra fields
+
+This meta data is specific to the "binary cache" family of Nix store types.
+This information is not intrinsic to the store object, but about how it is stored.
+
+* `url`:
+
+  Where to download a compressed archive of the file system objects of this store object.
+
+* `compression`:
+
+  The compression format that the archive is in.
+
+* `fileHash`:
+
+  A digest for the compressed archive itself, as opposed to the data contained within.
+
+* `fileSize`:
+
+  The size of the compressed archive itself.
+
+## Computed closure fields
+
+These fields are not stored at all, but computed by traverising the other other fields across all the store objects in a [closure].
+
+* `closureSize`:
+
+  The total size of the compressed archive itself for this object, and the compressed archive of every object in this object's [closure].
+
+### `.narinfo` extra fields
+
+* `closureSize`:
+
+  The total size of this store object and every other object in its [closure].
+
+[closure]: @docroot@/glossary.md#gloss-closure
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 49a4c1f2a..3107c8aed 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -635,7 +635,7 @@ public:
 
           - the store object has been signed using a key in the trusted keys list
           - the [`require-sigs`](#conf-require-sigs) option has been set to `false`
-          - the store object is [output-addressed](@docroot@/glossary.md#gloss-output-addressed-store-object)
+          - the store object is [content-addressed](@docroot@/glossary.md#gloss-content-addressed-store-object)
         )",
         {"binary-cache-public-keys"}};
 
diff --git a/src/nix/derivation-add.md b/src/nix/derivation-add.md
index f116681ab..d9b8467df 100644
--- a/src/nix/derivation-add.md
+++ b/src/nix/derivation-add.md
@@ -9,10 +9,11 @@ Store derivations are used internally by Nix. They are store paths with
 extension `.drv` that represent the build-time dependency graph to which
 a Nix expression evaluates.
 
-[store derivation]: ../../glossary.md#gloss-store-derivation
 
-The JSON format is documented under the [`derivation show`] command.
+[store derivation]: @docroot@/glossary.md#gloss-store-derivation
 
-[`derivation show`]: ./nix3-derivation-show.md
+`nix derivation add` takes a single derivation in the following format:
+
+{{#include ../../json/derivation.md}}
 
 )""
diff --git a/src/nix/derivation-show.md b/src/nix/derivation-show.md
index 1296e2885..884f1adc6 100644
--- a/src/nix/derivation-show.md
+++ b/src/nix/derivation-show.md
@@ -5,8 +5,6 @@ R""(
 * Show the [store derivation] that results from evaluating the Hello
   package:
 
-  [store derivation]: ../../glossary.md#gloss-store-derivation
-
   ```console
   # nix derivation show nixpkgs#hello
   {
@@ -48,62 +46,12 @@ a Nix expression evaluates.
 By default, this command only shows top-level derivations, but with
 `--recursive`, it also shows their dependencies.
 
-The JSON output is a JSON object whose keys are the store paths of the
-derivations, and whose values are a JSON object with the following
-fields:
+[store derivation]: @docroot@/glossary.md#gloss-store-derivation
 
-* `name`: The name of the derivation. This is used when calculating the
-  store paths of the derivation's outputs.
+`nix derivation show` outputs a JSON map of [store path]s to derivations in the following format:
 
-* `outputs`: Information about the output paths of the
-  derivation. This is a JSON object with one member per output, where
-  the key is the output name and the value is a JSON object with these
-  fields:
+[store path]: @docroot@/glossary.md#gloss-store-path
 
-  * `path`: The output path.
-  * `hashAlgo`: For fixed-output derivations, the hashing algorithm
-    (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a
-    NAR hash rather than a flat file hash.
-  * `hash`: For fixed-output derivations, the expected content hash in
-    base-16.
-
-  Example:
-
-  ```json
-  "outputs": {
-    "out": {
-      "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source",
-      "hashAlgo": "r:sha256",
-      "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62"
-    }
-  }
-  ```
-
-* `inputSrcs`: A list of store paths on which this derivation depends.
-
-* `inputDrvs`: A JSON object specifying the derivations on which this
-  derivation depends, and what outputs of those derivations. For
-  example,
-
-  ```json
-  "inputDrvs": {
-    "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"],
-    "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"]
-  }
-  ```
-
-  specifies that this derivation depends on the `dev` output of
-  `curl`, and the `out` output of `unzip`.
-
-* `system`: The system type on which this derivation is to be built
-  (e.g. `x86_64-linux`).
-
-* `builder`: The absolute path of the program to be executed to run
-  the build. Typically this is the `bash` shell
-  (e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`).
-
-* `args`: The command-line arguments passed to the `builder`.
-
-* `env`: The environment passed to the `builder`.
+{{#include ../../json/derivation.md}}
 
 )""

From 65294fe5fe4fd5419ea374e73710e8a217ba8060 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sat, 20 Jan 2024 17:07:21 -0500
Subject: [PATCH 419/421] Fix typo in upcomming release notes

Thanks @cole-h for finding in https://github.com/NixOS/nix/pull/9815#discussion_r1460604130
---
 doc/manual/rl-next/nix-store-add.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual/rl-next/nix-store-add.md b/doc/manual/rl-next/nix-store-add.md
index d55711569..5ef2913b4 100644
--- a/doc/manual/rl-next/nix-store-add.md
+++ b/doc/manual/rl-next/nix-store-add.md
@@ -4,4 +4,4 @@ prs: 9809
 ---
 
 Adds a missing feature that was present in the old CLI, and matches our
-plans to have similar flags for `nix hash convert` and `hash hash path`.
+plans to have similar flags for `nix hash convert` and `nix hash path`.

From 202c5e2afc14232b3c9ff32b014387d76c45b3d7 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 19 Jan 2024 20:14:48 -0500
Subject: [PATCH 420/421] Start standardizing hash algo flags

Do this if we want to do `--hash-algo` everywhere, and not `--algo` for
hash commands.

The new `nix hash convert` is updated. Deprecated new CLI commands are
left as-is (`nix hash path` needs to be redone and is also left as-is).
---
 doc/manual/rl-next/nix-hash-convert.md | 12 +++++-----
 src/libutil/args.hh                    |  6 +++++
 src/nix/add-to-store.cc                |  2 +-
 src/nix/hash.cc                        |  2 +-
 tests/functional/hash.sh               | 32 +++++++++++++-------------
 5 files changed, 30 insertions(+), 24 deletions(-)

diff --git a/doc/manual/rl-next/nix-hash-convert.md b/doc/manual/rl-next/nix-hash-convert.md
index 2b718a66b..69db9508a 100644
--- a/doc/manual/rl-next/nix-hash-convert.md
+++ b/doc/manual/rl-next/nix-hash-convert.md
@@ -9,7 +9,7 @@ to stabilization! Examples:
 - Convert the hash to `nix32`.
 
   ```bash
-  $ nix hash convert --algo "sha1" --to nix32 "800d59cfcd3c05e900cb4e214be48f6b886a08df"
+  $ nix hash convert --hash-algo "sha1" --to nix32 "800d59cfcd3c05e900cb4e214be48f6b886a08df"
   vw46m23bizj4n8afrc0fj19wrp7mj3c0
   ```
   `nix32` is a base32 encoding with a nix-specific character set.
@@ -17,23 +17,23 @@ to stabilization! Examples:
   hash.
 - Convert the hash to the `sri` format that includes an algorithm specification:
   ```bash
-  nix hash convert --algo "sha1" "800d59cfcd3c05e900cb4e214be48f6b886a08df"
+  nix hash convert --hash-algo "sha1" "800d59cfcd3c05e900cb4e214be48f6b886a08df"
   sha1-gA1Zz808BekAy04hS+SPa4hqCN8=
   ```
   or with an explicit `-to` format:
   ```bash
-  nix hash convert --algo "sha1" --to sri "800d59cfcd3c05e900cb4e214be48f6b886a08df"
+  nix hash convert --hash-algo "sha1" --to sri "800d59cfcd3c05e900cb4e214be48f6b886a08df"
   sha1-gA1Zz808BekAy04hS+SPa4hqCN8=
   ```
 - Assert the input format of the hash:
   ```bash
-  nix hash convert --algo "sha256" --from nix32 "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="
+  nix hash convert --hash-algo "sha256" --from nix32 "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="
   error: input hash 'ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=' does not have the expected format '--from nix32'
-  nix hash convert --algo "sha256" --from nix32 "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s"
+  nix hash convert --hash-algo "sha256" --from nix32 "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s"
   sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=
   ```
 
-The `--to`/`--from`/`--algo` parameters have context-sensitive auto-completion.
+The `--to`/`--from`/`--hash-algo` parameters have context-sensitive auto-completion.
 
 ## Related Deprecations
 
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 18b0ae583..6c9c48065 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -177,7 +177,13 @@ protected:
         std::optional<ExperimentalFeature> experimentalFeature;
 
         static Flag mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha);
+        static Flag mkHashAlgoFlag(HashAlgorithm * ha) {
+            return mkHashAlgoFlag("hash-algo", ha);
+        }
         static Flag mkHashAlgoOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha);
+        static Flag mkHashAlgoOptFlag(std::optional<HashAlgorithm> * oha) {
+            return mkHashAlgoOptFlag("hash-algo", oha);
+        }
         static Flag mkHashFormatFlagWithDefault(std::string && longName, HashFormat * hf);
         static Flag mkHashFormatOptFlag(std::string && longName, std::optional<HashFormat> * ohf);
     };
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index f2dbe8a2c..7c534517d 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -53,7 +53,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
             }},
         });
 
-        addFlag(Flag::mkHashAlgoFlag("hash-algo", &hashAlgo));
+        addFlag(Flag::mkHashAlgoFlag(&hashAlgo));
     }
 
     void run(ref<Store> store) override
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 83694306e..8ab89e433 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -141,7 +141,7 @@ struct CmdHashConvert : Command
     CmdHashConvert(): to(HashFormat::SRI) {
         addFlag(Args::Flag::mkHashFormatOptFlag("from", &from));
         addFlag(Args::Flag::mkHashFormatFlagWithDefault("to", &to));
-        addFlag(Args::Flag::mkHashAlgoOptFlag("algo", &algo));
+        addFlag(Args::Flag::mkHashAlgoOptFlag(&algo));
         expectArgs({
            .label = "hashes",
            .handler = {&hashStrings},
diff --git a/tests/functional/hash.sh b/tests/functional/hash.sh
index 47eed5178..ff270076e 100644
--- a/tests/functional/hash.sh
+++ b/tests/functional/hash.sh
@@ -87,7 +87,7 @@ try3() {
     # $2 = expected hash in base16
     # $3 = expected hash in base32
     # $4 = expected hash in base64
-    h64=$(nix hash convert --algo "$1" --to base64 "$2")
+    h64=$(nix hash convert --hash-algo "$1" --to base64 "$2")
     [ "$h64" = "$4" ]
     h64=$(nix-hash --type "$1" --to-base64 "$2")
     [ "$h64" = "$4" ]
@@ -95,13 +95,13 @@ try3() {
     h64=$(nix hash to-base64 --type "$1" "$2")
     [ "$h64" = "$4" ]
 
-    sri=$(nix hash convert --algo "$1" --to sri "$2")
+    sri=$(nix hash convert --hash-algo "$1" --to sri "$2")
     [ "$sri" = "$1-$4" ]
     sri=$(nix-hash --type "$1" --to-sri "$2")
     [ "$sri" = "$1-$4" ]
     sri=$(nix hash to-sri --type "$1" "$2")
     [ "$sri" = "$1-$4" ]
-    h32=$(nix hash convert --algo "$1" --to base32 "$2")
+    h32=$(nix hash convert --hash-algo "$1" --to base32 "$2")
     [ "$h32" = "$3" ]
     h32=$(nix-hash --type "$1" --to-base32 "$2")
     [ "$h32" = "$3" ]
@@ -110,7 +110,7 @@ try3() {
     h16=$(nix-hash --type "$1" --to-base16 "$h32")
     [ "$h16" = "$2" ]
 
-    h16=$(nix hash convert --algo "$1" --to base16 "$h64")
+    h16=$(nix hash convert --hash-algo "$1" --to base16 "$h64")
     [ "$h16" = "$2" ]
     h16=$(nix hash to-base16 --type "$1" "$h64")
     [ "$h16" = "$2" ]
@@ -143,40 +143,40 @@ try3() {
     # Auto-detecting the input from algo and length.
     #
 
-    sri=$(nix hash convert --algo "$1" "$2")
+    sri=$(nix hash convert --hash-algo "$1" "$2")
     [ "$sri" = "$1-$4" ]
-    sri=$(nix hash convert --algo "$1" "$3")
+    sri=$(nix hash convert --hash-algo "$1" "$3")
     [ "$sri" = "$1-$4" ]
-    sri=$(nix hash convert --algo "$1" "$4")
+    sri=$(nix hash convert --hash-algo "$1" "$4")
     [ "$sri" = "$1-$4" ]
 
-    sri=$(nix hash convert --algo "$1" "$2")
+    sri=$(nix hash convert --hash-algo "$1" "$2")
     [ "$sri" = "$1-$4" ]
-    sri=$(nix hash convert --algo "$1" "$3")
+    sri=$(nix hash convert --hash-algo "$1" "$3")
     [ "$sri" = "$1-$4" ]
-    sri=$(nix hash convert --algo "$1" "$4")
+    sri=$(nix hash convert --hash-algo "$1" "$4")
     [ "$sri" = "$1-$4" ]
 
     #
     # Asserting input format succeeds.
     #
 
-    sri=$(nix hash convert --algo "$1" --from base16 "$2")
+    sri=$(nix hash convert --hash-algo "$1" --from base16 "$2")
     [ "$sri" = "$1-$4" ]
-    sri=$(nix hash convert --algo "$1" --from nix32 "$3")
+    sri=$(nix hash convert --hash-algo "$1" --from nix32 "$3")
     [ "$sri" = "$1-$4" ]
-    sri=$(nix hash convert --algo "$1" --from base64 "$4")
+    sri=$(nix hash convert --hash-algo "$1" --from base64 "$4")
     [ "$sri" = "$1-$4" ]
 
     #
     # Asserting input format fails.
     #
 
-    fail=$(nix hash convert --algo "$1" --from nix32 "$2" 2>&1 || echo "exit: $?")
+    fail=$(nix hash convert --hash-algo "$1" --from nix32 "$2" 2>&1 || echo "exit: $?")
     [[ "$fail" == *"error: input hash"*"exit: 1" ]]
-    fail=$(nix hash convert --algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?")
+    fail=$(nix hash convert --hash-algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?")
     [[ "$fail" == *"error: input hash"*"exit: 1" ]]
-    fail=$(nix hash convert --algo "$1" --from nix32 "$4" 2>&1 || echo "exit: $?")
+    fail=$(nix hash convert --hash-algo "$1" --from nix32 "$4" 2>&1 || echo "exit: $?")
     [[ "$fail" == *"error: input hash"*"exit: 1" ]]
 
 }

From 316e50cc7c0bad8448c9f475993e52f9d5dee7c0 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 22 Jan 2024 10:32:25 -0500
Subject: [PATCH 421/421] Fix `if`...`if`...`else` ambiguity

This can be parsed two ways. Add a pair of braces so it must be parsed
the intended way.
---
 src/libexpr/primops/fetchTree.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index bc5a69720..d32c264f7 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -174,11 +174,12 @@ static void fetchTree(
     if (!evalSettings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes))
         input = lookupInRegistries(state.store, input).first;
 
-    if (evalSettings.pureEval && !input.isLocked())
+    if (evalSettings.pureEval && !input.isLocked()) {
         if (params.isFetchGit)
             state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchGit' requires a locked input, at %s", state.positions[pos]));
         else
             state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", state.positions[pos]));
+    }
 
     state.checkURI(input.toURLString());