From 188d97e1f1a6ce41f1eaed813adf878cfa6acdeb Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 15 Oct 2024 20:55:05 +0200
Subject: [PATCH 01/14] Restore input substitution

The ability to substitute inputs was removed in #10612 because it was
broken: with user-specified inputs containing a `narHash` attribute,
substitution resulted in an input that lacked the attributes returned
by the real fetcher (such as `lastModified`).

To fix this, we introduce a new input attribute `final`. If `final =
true`, fetching the input cannot add or change any attributes.

We only attempt to substitute inputs that have `final = true`. This is
implied by lock file entries; we only write a lock file if all its
entries are "final".

The user can specified `final = true` in `fetchTree`, in which case it
is their responsibility to ensure that all attributes returned by the
fetcher are included in the `fetchTree` call. For example,

  nix eval --impure --expr 'builtins.fetchTree { type = "github"; owner = "NixOS"; repo = "patchelf"; final = true; narHash = "sha256-FSoxTcRZMGHNJh8dNtKOkcUtjhmhU6yQXcZZfUPLhQM="; }'

succeeds in a store path with the specified NAR hash exists or is
substitutable, but fails with

  error: fetching final input '{"final":true,"narHash":"sha256-FSoxTcRZMGHNJh8dNtKOkcUtjhmhU6yQXcZZfUPLhQM=","owner":"NixOS","repo":"patchelf","type":"github"}' resulted in different input '{"final":true,"lastModified":1718457448,"narHash":"sha256-FSoxTcRZMGHNJh8dNtKOkcUtjhmhU6yQXcZZfUPLhQM=","owner":"NixOS","repo":"patchelf","rev":"a0f54334df36770b335c051e540ba40afcbf8378","type":"github"}'
---
 src/libexpr/call-flake.nix     |  3 ++-
 src/libfetchers/fetchers.cc    | 46 +++++++++++++++++++++++++++++++++-
 src/libfetchers/fetchers.hh    | 33 +++++++++++++-----------
 src/libfetchers/path.cc        |  1 +
 src/libflake/flake/flake.cc    |  1 -
 src/libflake/flake/lockfile.cc | 12 +++++++--
 src/libflake/flake/lockfile.hh |  4 +--
 7 files changed, 79 insertions(+), 21 deletions(-)

diff --git a/src/libexpr/call-flake.nix b/src/libexpr/call-flake.nix
index a411564df..c44d64885 100644
--- a/src/libexpr/call-flake.nix
+++ b/src/libexpr/call-flake.nix
@@ -44,7 +44,8 @@ let
               overrides.${key}.sourceInfo
             else
               # FIXME: remove obsolete node.info.
-              fetchTree (node.info or {} // removeAttrs node.locked ["dir"]);
+              # Note: lock file entries are always final.
+              fetchTree (node.info or {} // removeAttrs node.locked ["dir"] // { final = true; });
 
           subdir = overrides.${key}.dir or node.locked.dir or "";
 
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index b07e8cb6e..ff4c7567f 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -3,6 +3,7 @@
 #include "source-path.hh"
 #include "fetch-to-store.hh"
 #include "json-utils.hh"
+#include "store-path-accessor.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -100,7 +101,7 @@ Input Input::fromAttrs(const Settings & settings, Attrs && attrs)
     auto allowedAttrs = inputScheme->allowedAttrs();
 
     for (auto & [name, _] : attrs)
-        if (name != "type" && allowedAttrs.count(name) == 0)
+        if (name != "type" && name != "final" && allowedAttrs.count(name) == 0)
             throw Error("input attribute '%s' not supported by scheme '%s'", name, schemeName);
 
     auto res = inputScheme->inputFromAttrs(settings, attrs);
@@ -145,6 +146,11 @@ bool Input::isLocked() const
     return scheme && scheme->isLocked(*this);
 }
 
+bool Input::isFinal() const
+{
+    return maybeGetBoolAttr(attrs, "final").value_or(false);
+}
+
 Attrs Input::toAttrs() const
 {
     return attrs;
@@ -221,6 +227,12 @@ void InputScheme::checkLocks(const Input & specified, const Input & final) const
             throw Error("'revCount' attribute mismatch in input '%s', expected %d",
                 final.to_string(), *prevRevCount);
     }
+
+    assert(final.isFinal());
+
+    if (specified.isFinal() && specified.attrs != final.attrs)
+        throw Error("fetching final input '%s' resulted in different input '%s'",
+            attrsToJSON(specified.attrs), attrsToJSON(final.attrs));
 }
 
 std::pair<ref<SourceAccessor>, Input> Input::getAccessor(ref<Store> store) const
@@ -244,11 +256,43 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
     if (!scheme)
         throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs()));
 
+    /* The tree may already be in the Nix store, or it could be
+       substituted (which is often faster than fetching from the
+       original source). So check that. We only do this for final
+       inputs, otherwise there is a risk that we don't return the
+       same attributes (like `lastModified`) that the "real" fetcher
+       would return.
+
+       FIXME: add a setting to disable this.
+       FIXME: substituting may be slower than fetching normally,
+       e.g. for fetchers like that Git that are incremental!
+    */
+    if (isFinal() && getNarHash()) {
+        try {
+            auto storePath = computeStorePath(*store);
+
+            store->ensurePath(storePath);
+
+            debug("using substituted/cached input '%s' in '%s'",
+                to_string(), store->printStorePath(storePath));
+
+            auto accessor = makeStorePathAccessor(store, storePath);
+
+            accessor->fingerprint = scheme->getFingerprint(store, *this);
+
+            return {accessor, *this};
+        } catch (Error & e) {
+            debug("substitution of input '%s' failed: %s", to_string(), e.what());
+        }
+    }
+
     auto [accessor, final] = scheme->getAccessor(store, *this);
 
     assert(!accessor->fingerprint);
     accessor->fingerprint = scheme->getFingerprint(store, final);
 
+    final.attrs.insert_or_assign("final", Explicit<bool>(true));
+
     return {accessor, std::move(final)};
 }
 
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index a5f9bdcc6..e74625f7f 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -84,11 +84,21 @@ public:
     bool isDirect() const;
 
     /**
-     * Check whether this is a "locked" input, that is,
-     * one that contains a commit hash or content hash.
+     * Check whether this is a "locked" input, that is, it has
+     * attributes like a Git revision or NAR hash that uniquely
+     * identify its contents.
      */
     bool isLocked() const;
 
+    /**
+     * Check whether this is a "final" input, meaning that fetching it
+     * will not add or change any attributes. For instance, a Git
+     * input with a `rev` attribute but without a `lastModified`
+     * attribute is considered locked but not final. Only "final"
+     * inputs can be substituted from a binary cache.
+     */
+    bool isFinal() const;
+
     bool operator ==(const Input & other) const noexcept;
 
     bool contains(const Input & other) const;
@@ -144,6 +154,10 @@ public:
     /**
      * For locked inputs, return a string that uniquely specifies the
      * content of the input (typically a commit hash or content hash).
+     *
+     * Only known-equivalent inputs should return the same fingerprint.
+     *
+     * This is not a stable identifier between Nix versions, but not guaranteed to change either.
      */
     std::optional<std::string> getFingerprint(ref<Store> store) const;
 };
@@ -212,24 +226,15 @@ struct InputScheme
      */
     virtual std::optional<ExperimentalFeature> experimentalFeature() const;
 
+    /// See `Input::isDirect()`.
     virtual bool isDirect(const Input & input) const
     { return true; }
 
-    /**
-     * A sufficiently unique string that can be used as a cache key to identify the `input`.
-     *
-     * Only known-equivalent inputs should return the same fingerprint.
-     *
-     * This is not a stable identifier between Nix versions, but not guaranteed to change either.
-     */
+    /// See `Input::getFingerprint()`.
     virtual std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const
     { return std::nullopt; }
 
-    /**
-     * Return `true` if this input is considered "locked", i.e. it has
-     * attributes like a Git revision or NAR hash that uniquely
-     * identify its contents.
-     */
+    /// See `Input::isLocked()`.
     virtual bool isLocked(const Input & input) const
     { return false; }
 
diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc
index fca0df84b..564ad6e71 100644
--- a/src/libfetchers/path.cc
+++ b/src/libfetchers/path.cc
@@ -72,6 +72,7 @@ struct PathInputScheme : InputScheme
         auto query = attrsToQuery(input.attrs);
         query.erase("path");
         query.erase("type");
+        query.erase("final");
         return ParsedURL {
             .scheme = "path",
             .path = getStrAttr(input.attrs, "path"),
diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc
index d18e01464..f6f29f241 100644
--- a/src/libflake/flake/flake.cc
+++ b/src/libflake/flake/flake.cc
@@ -85,7 +85,6 @@ static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos
         state.forceValue(value, pos);
 }
 
-
 static void expectType(EvalState & state, ValueType type,
     Value & value, const PosIdx pos)
 {
diff --git a/src/libflake/flake/lockfile.cc b/src/libflake/flake/lockfile.cc
index 70b60716f..f80c27acd 100644
--- a/src/libflake/flake/lockfile.cc
+++ b/src/libflake/flake/lockfile.cc
@@ -46,6 +46,10 @@ LockedNode::LockedNode(
     if (!lockedRef.input.isLocked())
         throw Error("lock file contains unlocked input '%s'",
             fetchers::attrsToJSON(lockedRef.input.toAttrs()));
+
+    // For backward compatibility, lock file entries are implicitly final.
+    assert(!lockedRef.input.attrs.contains("final"));
+    lockedRef.input.attrs.insert_or_assign("final", Explicit<bool>(true));
 }
 
 StorePath LockedNode::computeStorePath(Store & store) const
@@ -53,7 +57,6 @@ StorePath LockedNode::computeStorePath(Store & store) const
     return lockedRef.input.computeStorePath(store);
 }
 
-
 static std::shared_ptr<Node> doFind(const ref<Node> & root, const InputPath & path, std::vector<InputPath> & visited)
 {
     auto pos = root;
@@ -191,6 +194,11 @@ std::pair<nlohmann::json, LockFile::KeyMap> LockFile::toJSON() const
         if (auto lockedNode = node.dynamic_pointer_cast<const LockedNode>()) {
             n["original"] = fetchers::attrsToJSON(lockedNode->originalRef.toAttrs());
             n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs());
+            /* For backward compatibility, omit the "final"
+               attribute. We never allow non-final inputs in lock files
+               anyway. */
+            assert(lockedNode->lockedRef.input.isFinal());
+            n["locked"].erase("final");
             if (!lockedNode->isFlake)
                 n["flake"] = false;
         }
@@ -239,7 +247,7 @@ std::optional<FlakeRef> LockFile::isUnlocked() const
     for (auto & i : nodes) {
         if (i == ref<const Node>(root)) continue;
         auto node = i.dynamic_pointer_cast<const LockedNode>();
-        if (node && !node->lockedRef.input.isLocked())
+        if (node && (!node->lockedRef.input.isLocked() || !node->lockedRef.input.isFinal()))
             return node->lockedRef;
     }
 
diff --git a/src/libflake/flake/lockfile.hh b/src/libflake/flake/lockfile.hh
index 841931c11..a2711a516 100644
--- a/src/libflake/flake/lockfile.hh
+++ b/src/libflake/flake/lockfile.hh
@@ -68,8 +68,8 @@ struct LockFile
     std::pair<std::string, KeyMap> to_string() const;
 
     /**
-     * Check whether this lock file has any unlocked inputs. If so,
-     * return one.
+     * Check whether this lock file has any unlocked or non-final
+     * inputs. If so, return one.
      */
     std::optional<FlakeRef> isUnlocked() const;
 

From fc09815eda00e3ba9211932ab14d2bdf4feab7db Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 16 Oct 2024 15:17:38 +0200
Subject: [PATCH 02/14] Typo

Co-authored-by: Cole Helbling <cole.e.helbling@outlook.com>
---
 src/libfetchers/fetchers.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index ff4c7567f..f25781a12 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -265,7 +265,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
 
        FIXME: add a setting to disable this.
        FIXME: substituting may be slower than fetching normally,
-       e.g. for fetchers like that Git that are incremental!
+       e.g. for fetchers like Git that are incremental!
     */
     if (isFinal() && getNarHash()) {
         try {

From ed1f9dd13f23450aad86f7687dd1b596d06ceed4 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 16 Oct 2024 15:18:23 +0200
Subject: [PATCH 03/14] Don't mark inputs as final in getAccessorUnchecked()

We haven't added the narHash attribute yet at that point. And if the
caller uses getAccesor() instead of fetchToStore() (e.g. in `nix
registry pin`), the narHash attribute will never be added. This could
lead to a mismatch.
---
 src/libfetchers/fetchers.cc | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index f25781a12..26229134d 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -185,6 +185,14 @@ std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
             auto narHash = store->queryPathInfo(storePath)->narHash;
             final.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
 
+            // FIXME: we would like to mark inputs as final in
+            // getAccessorUnchecked(), but then we can't add
+            // narHash. Or maybe narHash should be excluded from the
+            // concept of "final" inputs?
+            final.attrs.insert_or_assign("final", Explicit<bool>(true));
+
+            assert(final.isFinal());
+
             scheme->checkLocks(*this, final);
 
             return {storePath, final};
@@ -228,8 +236,6 @@ void InputScheme::checkLocks(const Input & specified, const Input & final) const
                 final.to_string(), *prevRevCount);
     }
 
-    assert(final.isFinal());
-
     if (specified.isFinal() && specified.attrs != final.attrs)
         throw Error("fetching final input '%s' resulted in different input '%s'",
             attrsToJSON(specified.attrs), attrsToJSON(final.attrs));
@@ -291,8 +297,6 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
     assert(!accessor->fingerprint);
     accessor->fingerprint = scheme->getFingerprint(store, final);
 
-    final.attrs.insert_or_assign("final", Explicit<bool>(true));
-
     return {accessor, std::move(final)};
 }
 

From 78b5b4c105f1adb33c416889f4378cede154cf68 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 17 Oct 2024 14:12:39 +0200
Subject: [PATCH 04/14] Tarball fetcher: Fix compat with old lock files that
 didn't include lastModified

Fixes flake-regressions/tests/DeterminateSystems/fh/0.1.10:

  error: fetching final input '{"final":true,"narHash":"sha256-0dZpggYjjmWEk+rGixiBHOHuQfLzEzNfrtjSig04s6Q=","rev":"9ccae1754eec0341b640d5705302ac0923d22875","revCount":1618,"type":"tarball","url":"https://api.flakehub.com/f/pinned/nix-community/fenix/0.1.1618%2Brev-9ccae1754eec0341b640d5705302ac0923d22875/018aea4c-03c9-7734-95d5-b84cc8881e3d/source.tar.gz"}' resulted in different input '{"final":true,"lastModified":1696141234,"narHash":"sha256-0dZpggYjjmWEk+rGixiBHOHuQfLzEzNfrtjSig04s6Q=","rev":"9ccae1754eec0341b640d5705302ac0923d22875","revCount":1618,"type":"tarball","url":"https://api.flakehub.com/f/pinned/nix-community/fenix/0.1.1618%2Brev-9ccae1754eec0341b640d5705302ac0923d22875/018aea4c-03c9-7734-95d5-b84cc8881e3d/source.tar.gz"}'
---
 src/libfetchers/tarball.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index 28574e7b1..27ad89b6e 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -384,7 +384,11 @@ struct TarballInputScheme : CurlInputScheme
             input = immutableInput;
         }
 
-        if (result.lastModified && !input.attrs.contains("lastModified"))
+        /* If we got a lastModified and the input is not final and
+           doesn't have one, then return it. Note that we don't do
+           this if the input is final for compatibility with old lock
+           files that didn't include lastModified. */
+        if (result.lastModified && !_input.isFinal() && !input.attrs.contains("lastModified"))
             input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
 
         input.attrs.insert_or_assign("narHash",

From 7d1f7f8d59fe1a9bbed3adc09a76de07ba84e8e8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 17 Oct 2024 16:20:08 +0200
Subject: [PATCH 05/14] Tarball fetcher: Handle lock files that *do* contain
 lastModified

Fixes flake-regressions/tests/DeterminateSystems/eva/0.1.0:

  error: 'lastModified' attribute mismatch in input 'https://api.flakehub.com/f/pinned/ipetkov/crane/0.14.1/018ac45c-ff5e-7076-b956-d478a0336516/source.tar.gz?narHash=sha256-mnE14re43v3/Jc50Jv0BKPMtEk7FEtDSligP6B5HwlI%3D', expected 1695511445
---
 src/libfetchers/fetchers.cc |  4 ++--
 src/libfetchers/tarball.cc  | 12 +++++++-----
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 26229134d..9717533d6 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -220,8 +220,8 @@ void InputScheme::checkLocks(const Input & specified, const Input & final) const
 
     if (auto prevLastModified = specified.getLastModified()) {
         if (final.getLastModified() != prevLastModified)
-            throw Error("'lastModified' attribute mismatch in input '%s', expected %d",
-                final.to_string(), *prevLastModified);
+            throw Error("'lastModified' attribute mismatch in input '%s', expected %d, got %d",
+                final.to_string(), *prevLastModified, final.getLastModified().value_or(-1));
     }
 
     if (auto prevRev = specified.getRev()) {
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index 27ad89b6e..e723d3061 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -384,11 +384,13 @@ struct TarballInputScheme : CurlInputScheme
             input = immutableInput;
         }
 
-        /* If we got a lastModified and the input is not final and
-           doesn't have one, then return it. Note that we don't do
-           this if the input is final for compatibility with old lock
-           files that didn't include lastModified. */
-        if (result.lastModified && !_input.isFinal() && !input.attrs.contains("lastModified"))
+        /* If we got a lastModified, then return it. But for
+           compatibility with old lock files that didn't include
+           lastModified, don't do this if the original input was final
+           and didn't contain a lastModified. */
+        if (result.lastModified
+            && !input.attrs.contains("lastModified")
+            && (!_input.isFinal() || _input.attrs.contains("lastModified")))
             input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
 
         input.attrs.insert_or_assign("narHash",

From a7a0767df7e97d6a4c1abbebda1f412e44fa9149 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 30 Oct 2024 20:53:41 +0100
Subject: [PATCH 06/14] Rename final -> __final

---
 src/libexpr/call-flake.nix     |  2 +-
 src/libfetchers/fetchers.cc    |  6 +++---
 src/libfetchers/fetchers.hh    | 13 +++++++------
 src/libfetchers/path.cc        |  2 +-
 src/libflake/flake/lockfile.cc |  8 ++++----
 5 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/src/libexpr/call-flake.nix b/src/libexpr/call-flake.nix
index c44d64885..79b3e804e 100644
--- a/src/libexpr/call-flake.nix
+++ b/src/libexpr/call-flake.nix
@@ -45,7 +45,7 @@ let
             else
               # FIXME: remove obsolete node.info.
               # Note: lock file entries are always final.
-              fetchTree (node.info or {} // removeAttrs node.locked ["dir"] // { final = true; });
+              fetchTree (node.info or {} // removeAttrs node.locked ["dir"] // { __final = true; });
 
           subdir = overrides.${key}.dir or node.locked.dir or "";
 
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 9717533d6..cea6e43ae 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -101,7 +101,7 @@ Input Input::fromAttrs(const Settings & settings, Attrs && attrs)
     auto allowedAttrs = inputScheme->allowedAttrs();
 
     for (auto & [name, _] : attrs)
-        if (name != "type" && name != "final" && allowedAttrs.count(name) == 0)
+        if (name != "type" && name != "__final" && allowedAttrs.count(name) == 0)
             throw Error("input attribute '%s' not supported by scheme '%s'", name, schemeName);
 
     auto res = inputScheme->inputFromAttrs(settings, attrs);
@@ -148,7 +148,7 @@ bool Input::isLocked() const
 
 bool Input::isFinal() const
 {
-    return maybeGetBoolAttr(attrs, "final").value_or(false);
+    return maybeGetBoolAttr(attrs, "__final").value_or(false);
 }
 
 Attrs Input::toAttrs() const
@@ -189,7 +189,7 @@ std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
             // getAccessorUnchecked(), but then we can't add
             // narHash. Or maybe narHash should be excluded from the
             // concept of "final" inputs?
-            final.attrs.insert_or_assign("final", Explicit<bool>(true));
+            final.attrs.insert_or_assign("__final", Explicit<bool>(true));
 
             assert(final.isFinal());
 
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index e74625f7f..ce535e6fe 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -78,24 +78,28 @@ public:
     Attrs toAttrs() const;
 
     /**
-     * Check whether this is a "direct" input, that is, not
+     * Return whether this is a "direct" input, that is, not
      * one that goes through a registry.
      */
     bool isDirect() const;
 
     /**
-     * Check whether this is a "locked" input, that is, it has
+     * Return whether this is a "locked" input, that is, it has
      * attributes like a Git revision or NAR hash that uniquely
      * identify its contents.
      */
     bool isLocked() const;
 
     /**
-     * Check whether this is a "final" input, meaning that fetching it
+     * Return whether this is a "final" input, meaning that fetching it
      * will not add or change any attributes. For instance, a Git
      * input with a `rev` attribute but without a `lastModified`
      * attribute is considered locked but not final. Only "final"
      * inputs can be substituted from a binary cache.
+     *
+     * The "final" state is denoted by the presence of an attribute
+     * `__final = true`. This attribute is currently undocumented and
+     * for internal use only.
      */
     bool isFinal() const;
 
@@ -226,15 +230,12 @@ struct InputScheme
      */
     virtual std::optional<ExperimentalFeature> experimentalFeature() const;
 
-    /// See `Input::isDirect()`.
     virtual bool isDirect(const Input & input) const
     { return true; }
 
-    /// See `Input::getFingerprint()`.
     virtual std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const
     { return std::nullopt; }
 
-    /// See `Input::isLocked()`.
     virtual bool isLocked(const Input & input) const
     { return false; }
 
diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc
index 564ad6e71..f12c969af 100644
--- a/src/libfetchers/path.cc
+++ b/src/libfetchers/path.cc
@@ -72,7 +72,7 @@ struct PathInputScheme : InputScheme
         auto query = attrsToQuery(input.attrs);
         query.erase("path");
         query.erase("type");
-        query.erase("final");
+        query.erase("__final");
         return ParsedURL {
             .scheme = "path",
             .path = getStrAttr(input.attrs, "path"),
diff --git a/src/libflake/flake/lockfile.cc b/src/libflake/flake/lockfile.cc
index f80c27acd..668ed165f 100644
--- a/src/libflake/flake/lockfile.cc
+++ b/src/libflake/flake/lockfile.cc
@@ -48,8 +48,8 @@ LockedNode::LockedNode(
             fetchers::attrsToJSON(lockedRef.input.toAttrs()));
 
     // For backward compatibility, lock file entries are implicitly final.
-    assert(!lockedRef.input.attrs.contains("final"));
-    lockedRef.input.attrs.insert_or_assign("final", Explicit<bool>(true));
+    assert(!lockedRef.input.attrs.contains("__final"));
+    lockedRef.input.attrs.insert_or_assign("__final", Explicit<bool>(true));
 }
 
 StorePath LockedNode::computeStorePath(Store & store) const
@@ -194,11 +194,11 @@ std::pair<nlohmann::json, LockFile::KeyMap> LockFile::toJSON() const
         if (auto lockedNode = node.dynamic_pointer_cast<const LockedNode>()) {
             n["original"] = fetchers::attrsToJSON(lockedNode->originalRef.toAttrs());
             n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs());
-            /* For backward compatibility, omit the "final"
+            /* For backward compatibility, omit the "__final"
                attribute. We never allow non-final inputs in lock files
                anyway. */
             assert(lockedNode->lockedRef.input.isFinal());
-            n["locked"].erase("final");
+            n["locked"].erase("__final");
             if (!lockedNode->isFlake)
                 n["flake"] = false;
         }

From 5c49d0b5d235e1cbec31b3225c338781f6c7c506 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 1 Nov 2024 15:34:48 +0100
Subject: [PATCH 07/14] Handle final handling for old lock files with improper
 narHash fields

This fixes the error

  '{"__final":true,"lastModified":1686592866,"narHash":"sha256-riGg89eWhXJcPNrQGcSwTEEm7CGxWC06oSX44hajeMw","owner":"nixos","repo":"nixpkgs","rev":"0eeebd64de89e4163f4d3cf34ffe925a5cf67a05","type":"github"}' resulted in different input
  '{"__final":true,"lastModified":1686592866,"narHash":"sha256-riGg89eWhXJcPNrQGcSwTEEm7CGxWC06oSX44hajeMw=","owner":"nixos","repo":"nixpkgs","rev":"0eeebd64de89e4163f4d3cf34ffe925a5cf67a05","type":"github"}'

in flake-regressions/tests/nix-community/patsh/0.2.1 (note the lack of
a trailing '=' in the NAR hash in the lock file).
---
 src/libfetchers/fetchers.cc | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index cea6e43ae..cce1971ff 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -236,9 +236,22 @@ void InputScheme::checkLocks(const Input & specified, const Input & final) const
                 final.to_string(), *prevRevCount);
     }
 
-    if (specified.isFinal() && specified.attrs != final.attrs)
-        throw Error("fetching final input '%s' resulted in different input '%s'",
-            attrsToJSON(specified.attrs), attrsToJSON(final.attrs));
+    /* If the original input is final, then the result must be the
+       same (i.e. cannot remove, add or change fields. */
+    if (specified.isFinal()) {
+
+        /* Backwards compatibility hack: we had some lock files in the
+           past that 'narHash' fields with incorrect base-64
+           formatting (lacking the trailing '=', e.g. 'sha256-ri...Mw'
+           instead of ''sha256-ri...Mw='). So fix */
+        auto specified2 = specified;
+        if (auto prevNarHash = specified2.getNarHash())
+            specified2.attrs.insert_or_assign("narHash", prevNarHash->to_string(HashFormat::SRI, true));
+
+        if (specified2.attrs != final.attrs)
+            throw Error("fetching final input '%s' resulted in different input '%s'",
+                attrsToJSON(specified2.attrs), attrsToJSON(final.attrs));
+    }
 }
 
 std::pair<ref<SourceAccessor>, Input> Input::getAccessor(ref<Store> store) const

From f314e35b3726e7295eb39e70fd2d67e473c12ef8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 1 Nov 2024 16:33:51 +0100
Subject: [PATCH 08/14] Simplify "final" inputs

We now just check that the fetcher doesn't change any attributes in
the input, and return all the original attributes (i.e. discarding any
new attributes and keeping any attributes that the fetcher didn't
keep).
---
 src/libfetchers/fetchers.cc | 50 ++++++++++++++++++++++---------------
 src/libfetchers/fetchers.hh | 27 ++++++++++++--------
 src/libfetchers/tarball.cc  | 12 ++++-----
 3 files changed, 51 insertions(+), 38 deletions(-)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index cce1971ff..2aedb8a2e 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -193,7 +193,7 @@ std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
 
             assert(final.isFinal());
 
-            scheme->checkLocks(*this, final);
+            checkLocks(*this, final);
 
             return {storePath, final};
         } catch (Error & e) {
@@ -205,8 +205,35 @@ std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
     return {std::move(storePath), input};
 }
 
-void InputScheme::checkLocks(const Input & specified, const Input & final) const
+void Input::checkLocks(Input specified, Input & final)
 {
+    /* If the original input is final, then we just return the
+       original attributes, dropping any new fields returned by the
+       fetcher. However, any fields that are in both the original and
+       final input must be identical. */
+    if (specified.isFinal()) {
+
+        /* Backwards compatibility hack: we had some lock files in the
+           past that 'narHash' fields with incorrect base-64
+           formatting (lacking the trailing '=', e.g. 'sha256-ri...Mw'
+           instead of ''sha256-ri...Mw='). So fix that. */
+        if (auto prevNarHash = specified.getNarHash())
+            specified.attrs.insert_or_assign("narHash", prevNarHash->to_string(HashFormat::SRI, true));
+
+        for (auto & field : specified.attrs) {
+            auto field2 = final.attrs.find(field.first);
+            if (field2 != final.attrs.end() && field.second != field2->second)
+                throw Error("mismatch in field '%s' of final input '%s', got '%s'",
+                    field.first,
+                    attrsToJSON(specified.attrs),
+                    attrsToJSON(final.attrs));
+        }
+
+        final.attrs = specified.attrs;
+
+        return;
+    }
+
     if (auto prevNarHash = specified.getNarHash()) {
         if (final.getNarHash() != prevNarHash) {
             if (final.getNarHash())
@@ -235,23 +262,6 @@ void InputScheme::checkLocks(const Input & specified, const Input & final) const
             throw Error("'revCount' attribute mismatch in input '%s', expected %d",
                 final.to_string(), *prevRevCount);
     }
-
-    /* If the original input is final, then the result must be the
-       same (i.e. cannot remove, add or change fields. */
-    if (specified.isFinal()) {
-
-        /* Backwards compatibility hack: we had some lock files in the
-           past that 'narHash' fields with incorrect base-64
-           formatting (lacking the trailing '=', e.g. 'sha256-ri...Mw'
-           instead of ''sha256-ri...Mw='). So fix */
-        auto specified2 = specified;
-        if (auto prevNarHash = specified2.getNarHash())
-            specified2.attrs.insert_or_assign("narHash", prevNarHash->to_string(HashFormat::SRI, true));
-
-        if (specified2.attrs != final.attrs)
-            throw Error("fetching final input '%s' resulted in different input '%s'",
-                attrsToJSON(specified2.attrs), attrsToJSON(final.attrs));
-    }
 }
 
 std::pair<ref<SourceAccessor>, Input> Input::getAccessor(ref<Store> store) const
@@ -259,7 +269,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessor(ref<Store> store) const
     try {
         auto [accessor, final] = getAccessorUnchecked(store);
 
-        scheme->checkLocks(*this, final);
+        checkLocks(*this, final);
 
         return {accessor, std::move(final)};
     } catch (Error & e) {
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index ce535e6fe..430d6e943 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -91,9 +91,9 @@ public:
     bool isLocked() const;
 
     /**
-     * Return whether this is a "final" input, meaning that fetching it
-     * will not add or change any attributes. For instance, a Git
-     * input with a `rev` attribute but without a `lastModified`
+     * Return whether this is a "final" input, meaning that fetching
+     * it will not add, remove or change any attributes. For instance,
+     * a Git input with a `rev` attribute but without a `lastModified`
      * attribute is considered locked but not final. Only "final"
      * inputs can be substituted from a binary cache.
      *
@@ -113,6 +113,19 @@ public:
      */
     std::pair<StorePath, Input> fetchToStore(ref<Store> store) const;
 
+    /**
+     * Check the locking attributes in `final` against
+     * `specified`. E.g. if `specified` has a `rev` attribute, then
+     * `final` must have the same `rev` attribute. Throw an exception
+     * if there is a mismatch.
+     *
+     * If `specified` is marked final (i.e. has the `__final`
+     * attribute), then the intersection of attributes in `specified`
+     * and `final` must be equal, and `final.attrs` is set to
+     * `specified.attrs` (i.e. we discard any new attributes).
+     */
+    static void checkLocks(Input specified, Input & final);
+
     /**
      * Return a `SourceAccessor` that allows access to files in the
      * input without copying it to the store. Also return a possibly
@@ -238,14 +251,6 @@ struct InputScheme
 
     virtual bool isLocked(const Input & input) const
     { return false; }
-
-    /**
-     * Check the locking attributes in `final` against
-     * `specified`. E.g. if `specified` has a `rev` attribute, then
-     * `final` must have the same `rev` attribute. Throw an exception
-     * if there is a mismatch.
-     */
-    virtual void checkLocks(const Input & specified, const Input & final) const;
 };
 
 void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index e723d3061..27ad89b6e 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -384,13 +384,11 @@ struct TarballInputScheme : CurlInputScheme
             input = immutableInput;
         }
 
-        /* If we got a lastModified, then return it. But for
-           compatibility with old lock files that didn't include
-           lastModified, don't do this if the original input was final
-           and didn't contain a lastModified. */
-        if (result.lastModified
-            && !input.attrs.contains("lastModified")
-            && (!_input.isFinal() || _input.attrs.contains("lastModified")))
+        /* If we got a lastModified and the input is not final and
+           doesn't have one, then return it. Note that we don't do
+           this if the input is final for compatibility with old lock
+           files that didn't include lastModified. */
+        if (result.lastModified && !_input.isFinal() && !input.attrs.contains("lastModified"))
             input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
 
         input.attrs.insert_or_assign("narHash",

From a150798ce48d457290dc00b7fe1b6a766ed4cc58 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Nov 2024 13:05:37 +0100
Subject: [PATCH 09/14] Document "final" semantics

---
 src/nix/flake.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/nix/flake.md b/src/nix/flake.md
index 2b999431c..b3321441c 100644
--- a/src/nix/flake.md
+++ b/src/nix/flake.md
@@ -666,6 +666,11 @@ following fields:
   other attributes are necessary because they provide information not
   stored in the store path.
 
+  The attributes in `locked` are considered "final", meaning that they are the only ones that are passed via the arguments of the `outputs` function of a flake.
+  For instance, if `locked` contains a `lastModified` attribute while the fetcher does not return a `lastModified` attribute, then the `lastModified` attribute will be passed to the `outputs` function.
+  Conversely, if `locked` does *not* contain a `lastModified` attribute while the fetcher *does* return a `lastModified` attribute, then no `lastModified` attribute will be passed.
+  If `locked` contains a `lastModifed` attribute and the fetcher returns a `lastModified` attribute, then they must have the same value.
+
 * `flake`: A Boolean denoting whether this is a flake or non-flake
   dependency. Corresponds to the `flake` attribute in the `inputs`
   attribute in `flake.nix`.

From 0401e2710f280db356fb606216287a56900462e5 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Nov 2024 13:12:02 +0100
Subject: [PATCH 10/14] More docs

---
 src/libfetchers/fetchers.hh | 7 +++----
 src/nix/flake.md            | 4 +++-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index 430d6e943..1ea75ba84 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -92,10 +92,9 @@ public:
 
     /**
      * Return whether this is a "final" input, meaning that fetching
-     * it will not add, remove or change any attributes. For instance,
-     * a Git input with a `rev` attribute but without a `lastModified`
-     * attribute is considered locked but not final. Only "final"
-     * inputs can be substituted from a binary cache.
+     * it will not add, remove or change any attributes. (See
+     * `checkLocks()` for the semantics.) Only "final" inputs can be
+     * substituted from a binary cache.
      *
      * The "final" state is denoted by the presence of an attribute
      * `__final = true`. This attribute is currently undocumented and
diff --git a/src/nix/flake.md b/src/nix/flake.md
index b3321441c..e35b936be 100644
--- a/src/nix/flake.md
+++ b/src/nix/flake.md
@@ -148,7 +148,7 @@ reference types:
 
 * `ref`: A Git or Mercurial branch or tag name.
 
-Finally, some attribute are typically not specified by the user, but
+Finally, some attributes are typically not specified by the user, but
 can occur in *locked* flake references and are available to Nix code:
 
 * `revCount`: The number of ancestors of the commit `rev`.
@@ -159,6 +159,8 @@ can occur in *locked* flake references and are available to Nix code:
   for tarball flakes, it's the most recent timestamp of any file
   inside the tarball.
 
+Attributes that start with `__` are internal or experimental and may be removed in future versions.
+
 ## Types
 
 Currently the `type` attribute can be one of the following:

From b7882d51f2da83d8062bbb0e353b41009bd25fcf Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 6 Nov 2024 13:19:53 +0100
Subject: [PATCH 11/14] Rename argument "final" to "result" to avoid ambiguity

---
 src/libfetchers/fetchers.cc | 60 ++++++++++++++++++-------------------
 src/libfetchers/fetchers.hh |  8 ++---
 2 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 2aedb8a2e..5c06a6bcb 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -178,24 +178,24 @@ std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
 
     auto [storePath, input] = [&]() -> std::pair<StorePath, Input> {
         try {
-            auto [accessor, final] = getAccessorUnchecked(store);
+            auto [accessor, result] = getAccessorUnchecked(store);
 
-            auto storePath = nix::fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, final.getName());
+            auto storePath = nix::fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, result.getName());
 
             auto narHash = store->queryPathInfo(storePath)->narHash;
-            final.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
+            result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
 
             // FIXME: we would like to mark inputs as final in
             // getAccessorUnchecked(), but then we can't add
             // narHash. Or maybe narHash should be excluded from the
             // concept of "final" inputs?
-            final.attrs.insert_or_assign("__final", Explicit<bool>(true));
+            result.attrs.insert_or_assign("__final", Explicit<bool>(true));
 
-            assert(final.isFinal());
+            assert(result.isFinal());
 
-            checkLocks(*this, final);
+            checkLocks(*this, result);
 
-            return {storePath, final};
+            return {storePath, result};
         } catch (Error & e) {
             e.addTrace({}, "while fetching the input '%s'", to_string());
             throw;
@@ -205,12 +205,12 @@ std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
     return {std::move(storePath), input};
 }
 
-void Input::checkLocks(Input specified, Input & final)
+void Input::checkLocks(Input specified, Input & result)
 {
     /* If the original input is final, then we just return the
        original attributes, dropping any new fields returned by the
-       fetcher. However, any fields that are in both the original and
-       final input must be identical. */
+       fetcher. However, any fields that are in both the specified and
+       result input must be identical. */
     if (specified.isFinal()) {
 
         /* Backwards compatibility hack: we had some lock files in the
@@ -221,24 +221,24 @@ void Input::checkLocks(Input specified, Input & final)
             specified.attrs.insert_or_assign("narHash", prevNarHash->to_string(HashFormat::SRI, true));
 
         for (auto & field : specified.attrs) {
-            auto field2 = final.attrs.find(field.first);
-            if (field2 != final.attrs.end() && field.second != field2->second)
-                throw Error("mismatch in field '%s' of final input '%s', got '%s'",
+            auto field2 = result.attrs.find(field.first);
+            if (field2 != result.attrs.end() && field.second != field2->second)
+                throw Error("mismatch in field '%s' of input '%s', got '%s'",
                     field.first,
                     attrsToJSON(specified.attrs),
-                    attrsToJSON(final.attrs));
+                    attrsToJSON(result.attrs));
         }
 
-        final.attrs = specified.attrs;
+        result.attrs = specified.attrs;
 
         return;
     }
 
     if (auto prevNarHash = specified.getNarHash()) {
-        if (final.getNarHash() != prevNarHash) {
-            if (final.getNarHash())
+        if (result.getNarHash() != prevNarHash) {
+            if (result.getNarHash())
                 throw Error((unsigned int) 102, "NAR hash mismatch in input '%s', expected '%s' but got '%s'",
-                    specified.to_string(), prevNarHash->to_string(HashFormat::SRI, true), final.getNarHash()->to_string(HashFormat::SRI, true));
+                    specified.to_string(), prevNarHash->to_string(HashFormat::SRI, true), result.getNarHash()->to_string(HashFormat::SRI, true));
             else
                 throw Error((unsigned int) 102, "NAR hash mismatch in input '%s', expected '%s' but got none",
                     specified.to_string(), prevNarHash->to_string(HashFormat::SRI, true));
@@ -246,32 +246,32 @@ void Input::checkLocks(Input specified, Input & final)
     }
 
     if (auto prevLastModified = specified.getLastModified()) {
-        if (final.getLastModified() != prevLastModified)
+        if (result.getLastModified() != prevLastModified)
             throw Error("'lastModified' attribute mismatch in input '%s', expected %d, got %d",
-                final.to_string(), *prevLastModified, final.getLastModified().value_or(-1));
+                result.to_string(), *prevLastModified, result.getLastModified().value_or(-1));
     }
 
     if (auto prevRev = specified.getRev()) {
-        if (final.getRev() != prevRev)
+        if (result.getRev() != prevRev)
             throw Error("'rev' attribute mismatch in input '%s', expected %s",
-                final.to_string(), prevRev->gitRev());
+                result.to_string(), prevRev->gitRev());
     }
 
     if (auto prevRevCount = specified.getRevCount()) {
-        if (final.getRevCount() != prevRevCount)
+        if (result.getRevCount() != prevRevCount)
             throw Error("'revCount' attribute mismatch in input '%s', expected %d",
-                final.to_string(), *prevRevCount);
+                result.to_string(), *prevRevCount);
     }
 }
 
 std::pair<ref<SourceAccessor>, Input> Input::getAccessor(ref<Store> store) const
 {
     try {
-        auto [accessor, final] = getAccessorUnchecked(store);
+        auto [accessor, result] = getAccessorUnchecked(store);
 
-        checkLocks(*this, final);
+        checkLocks(*this, result);
 
-        return {accessor, std::move(final)};
+        return {accessor, std::move(result)};
     } catch (Error & e) {
         e.addTrace({}, "while fetching the input '%s'", to_string());
         throw;
@@ -315,12 +315,12 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
         }
     }
 
-    auto [accessor, final] = scheme->getAccessor(store, *this);
+    auto [accessor, result] = scheme->getAccessor(store, *this);
 
     assert(!accessor->fingerprint);
-    accessor->fingerprint = scheme->getFingerprint(store, final);
+    accessor->fingerprint = scheme->getFingerprint(store, result);
 
-    return {accessor, std::move(final)};
+    return {accessor, std::move(result)};
 }
 
 Input Input::applyOverrides(
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index 1ea75ba84..b28ec4568 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -113,17 +113,17 @@ public:
     std::pair<StorePath, Input> fetchToStore(ref<Store> store) const;
 
     /**
-     * Check the locking attributes in `final` against
+     * Check the locking attributes in `result` against
      * `specified`. E.g. if `specified` has a `rev` attribute, then
-     * `final` must have the same `rev` attribute. Throw an exception
+     * `result` must have the same `rev` attribute. Throw an exception
      * if there is a mismatch.
      *
      * If `specified` is marked final (i.e. has the `__final`
      * attribute), then the intersection of attributes in `specified`
-     * and `final` must be equal, and `final.attrs` is set to
+     * and `result` must be equal, and `final.attrs` is set to
      * `specified.attrs` (i.e. we discard any new attributes).
      */
-    static void checkLocks(Input specified, Input & final);
+    static void checkLocks(Input specified, Input & result);
 
     /**
      * Return a `SourceAccessor` that allows access to files in the

From d90b56d52714426b2d7f946e2f75b1aaebff1221 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 8 Nov 2024 17:31:35 +0100
Subject: [PATCH 12/14] Remove no longer needed hack

---
 src/libfetchers/tarball.cc | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index 27ad89b6e..28574e7b1 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -384,11 +384,7 @@ struct TarballInputScheme : CurlInputScheme
             input = immutableInput;
         }
 
-        /* If we got a lastModified and the input is not final and
-           doesn't have one, then return it. Note that we don't do
-           this if the input is final for compatibility with old lock
-           files that didn't include lastModified. */
-        if (result.lastModified && !_input.isFinal() && !input.attrs.contains("lastModified"))
+        if (result.lastModified && !input.attrs.contains("lastModified"))
             input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
 
         input.attrs.insert_or_assign("narHash",

From 4dceca51deaffc7140a2b8a19d75c9cfa2d64c48 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 8 Nov 2024 19:27:54 +0100
Subject: [PATCH 13/14] Don't allow __final in fetchTree

It's now only allowed in fetchFinalTree, which is not exposed to users
but only to call-flake.nix.
---
 src/libexpr/call-flake.nix       |  5 ++++-
 src/libexpr/eval.cc              | 12 +++++++++---
 src/libexpr/eval.hh              | 10 ++++++++++
 src/libexpr/primops/fetchTree.cc | 20 ++++++++++++++++++++
 src/libflake/flake/flake.cc      |  8 +++++---
 src/libflake/flake/flake.hh      |  7 +++++++
 src/nix/flake.md                 |  2 --
 7 files changed, 55 insertions(+), 9 deletions(-)

diff --git a/src/libexpr/call-flake.nix b/src/libexpr/call-flake.nix
index 79b3e804e..a008346e5 100644
--- a/src/libexpr/call-flake.nix
+++ b/src/libexpr/call-flake.nix
@@ -10,6 +10,9 @@ lockFileStr:
 # unlocked trees.
 overrides:
 
+# This is `prim_fetchFinalTree`.
+fetchTreeFinal:
+
 let
 
   lockFile = builtins.fromJSON lockFileStr;
@@ -45,7 +48,7 @@ let
             else
               # FIXME: remove obsolete node.info.
               # Note: lock file entries are always final.
-              fetchTree (node.info or {} // removeAttrs node.locked ["dir"] // { __final = true; });
+              fetchTreeFinal (node.info or {} // removeAttrs node.locked ["dir"]);
 
           subdir = overrides.${key}.dir or node.locked.dir or "";
 
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index e4937735b..03d03e076 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -510,9 +510,15 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
 
     Value * v = allocValue();
     v->mkPrimOp(new PrimOp(primOp));
-    staticBaseEnv->vars.emplace_back(envName, baseEnvDispl);
-    baseEnv.values[baseEnvDispl++] = v;
-    baseEnv.values[0]->payload.attrs->push_back(Attr(symbols.create(primOp.name), v));
+
+    if (primOp.internal)
+        internalPrimOps.emplace(primOp.name, v);
+    else {
+        staticBaseEnv->vars.emplace_back(envName, baseEnvDispl);
+        baseEnv.values[baseEnvDispl++] = v;
+        baseEnv.values[0]->payload.attrs->push_back(Attr(symbols.create(primOp.name), v));
+    }
+
     return v;
 }
 
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index f7ed6be83..6b344137f 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -107,6 +107,11 @@ struct PrimOp
      */
     std::optional<ExperimentalFeature> experimentalFeature;
 
+    /**
+     * If true, this primop is not exposed to the user.
+     */
+    bool internal = false;
+
     /**
      * Validity check to be performed by functions that introduce primops,
      * such as RegisterPrimOp() and Value::mkPrimOp().
@@ -591,6 +596,11 @@ public:
      */
     std::shared_ptr<StaticEnv> staticBaseEnv; // !!! should be private
 
+    /**
+     * Internal primops not exposed to the user.
+     */
+    std::unordered_map<std::string, Value *, std::hash<std::string>, std::equal_to<std::string>, traceable_allocator<std::pair<const std::string, Value *>>> internalPrimOps;
+
     /**
      * Name and documentation about every constant.
      *
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index d2266e2bc..c207da8ad 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -78,6 +78,7 @@ struct FetchTreeParams {
     bool emptyRevFallback = false;
     bool allowNameArgument = false;
     bool isFetchGit = false;
+    bool isFinal = false;
 };
 
 static void fetchTree(
@@ -195,6 +196,13 @@ static void fetchTree(
 
     state.checkURI(input.toURLString());
 
+    if (params.isFinal) {
+        input.attrs.insert_or_assign("__final", Explicit<bool>(true));
+    } else {
+        if (input.isFinal())
+            throw Error("input '%s' is not allowed to use the '__final' attribute", input.to_string());
+    }
+
     auto [storePath, input2] = input.fetchToStore(state.store);
 
     state.allowPath(storePath);
@@ -431,6 +439,18 @@ static RegisterPrimOp primop_fetchTree({
     .experimentalFeature = Xp::FetchTree,
 });
 
+void prim_fetchFinalTree(EvalState & state, const PosIdx pos, Value * * args, Value & v)
+{
+    fetchTree(state, pos, args, v, {.isFinal = true});
+}
+
+static RegisterPrimOp primop_fetchFinalTree({
+    .name = "fetchFinalTree",
+    .args = {"input"},
+    .fun = prim_fetchFinalTree,
+    .internal = true,
+});
+
 static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v,
     const std::string & who, bool unpack, std::string name)
 {
diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc
index f6f29f241..edb76f861 100644
--- a/src/libflake/flake/flake.cc
+++ b/src/libflake/flake/flake.cc
@@ -809,12 +809,14 @@ void callFlake(EvalState & state,
     auto vCallFlake = state.allocValue();
     state.evalFile(state.callFlakeInternal, *vCallFlake);
 
-    auto vTmp1 = state.allocValue();
     auto vLocks = state.allocValue();
     vLocks->mkString(lockFileStr);
-    state.callFunction(*vCallFlake, *vLocks, *vTmp1, noPos);
 
-    state.callFunction(*vTmp1, vOverrides, vRes, noPos);
+    auto vFetchFinalTree = get(state.internalPrimOps, "fetchFinalTree");
+    assert(vFetchFinalTree);
+
+    Value * args[] = {vLocks, &vOverrides, *vFetchFinalTree};
+    state.callFunction(*vCallFlake, 3, args, vRes, noPos);
 }
 
 void initLib(const Settings & settings)
diff --git a/src/libflake/flake/flake.hh b/src/libflake/flake/flake.hh
index 496e18673..cc2bea76e 100644
--- a/src/libflake/flake/flake.hh
+++ b/src/libflake/flake/flake.hh
@@ -234,4 +234,11 @@ void emitTreeAttrs(
     bool emptyRevFallback = false,
     bool forceDirty = false);
 
+/**
+ * An internal builtin similar to `fetchTree`, except that it
+ * always treats the input as final (i.e. no attributes can be
+ * added/removed/changed).
+ */
+void prim_fetchFinalTree(EvalState & state, const PosIdx pos, Value * * args, Value & v);
+
 }
diff --git a/src/nix/flake.md b/src/nix/flake.md
index e35b936be..a9b703762 100644
--- a/src/nix/flake.md
+++ b/src/nix/flake.md
@@ -159,8 +159,6 @@ can occur in *locked* flake references and are available to Nix code:
   for tarball flakes, it's the most recent timestamp of any file
   inside the tarball.
 
-Attributes that start with `__` are internal or experimental and may be removed in future versions.
-
 ## Types
 
 Currently the `type` attribute can be one of the following:

From 036359ac84029ca24cca262b17e35d6b0b834e34 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Mon, 11 Nov 2024 13:58:12 +0100
Subject: [PATCH 14/14] Remove release note about flake substitution

---
 doc/manual/source/release-notes/rl-2.25.md | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/doc/manual/source/release-notes/rl-2.25.md b/doc/manual/source/release-notes/rl-2.25.md
index 2b6706496..29e3e509c 100644
--- a/doc/manual/source/release-notes/rl-2.25.md
+++ b/doc/manual/source/release-notes/rl-2.25.md
@@ -70,12 +70,6 @@
 
   Author: [**@zimbatm**](https://github.com/zimbatm)
 
-- Flakes are no longer substituted [#10612](https://github.com/NixOS/nix/pull/10612)
-
-  Nix will no longer attempt to substitute the source code of flakes from a binary cache. This functionality was broken because it could lead to different evaluation results depending on whether the flake was available in the binary cache, or even depending on whether the flake was already in the local store.
-
-  Author: [**@edolstra**](https://github.com/edolstra)
-
 - `<nix/fetchurl.nix>` uses TLS verification [#11585](https://github.com/NixOS/nix/pull/11585)
 
   Previously `<nix/fetchurl.nix>` did not do TLS verification. This was because the Nix sandbox in the past did not have access to TLS certificates, and Nix checks the hash of the fetched file anyway. However, this can expose authentication data from `netrc` and URLs to man-in-the-middle attackers. In addition, Nix now in some cases (such as when using impure derivations) does *not* check the hash. Therefore we have now enabled TLS verification. This means that downloads by `<nix/fetchurl.nix>` will now fail if you're fetching from a HTTPS server that does not have a valid certificate.