Merge pull request #11657 from DeterminateSystems/nix-copy-gc

nix copy: Add --profile and --out-link flags

doc/manual/rl-next/nix-copy-flags.md

@@ -0,0 +1,18 @@
+---
+synopsis: "`nix copy` supports `--profile` and `--out-link`"
+prs: [11657]
+---
+
+The `nix copy` command now has flags `--profile` and `--out-link`, similar to `nix build`. `--profile` makes a profile point to the
+top-level store path, while `--out-link` create symlinks to the top-level store paths.
+
+For example, when updating the local NixOS system profile from a NixOS system closure on a remote machine, instead of
+```
+# nix copy --from ssh://server $path
+# nix build --profile /nix/var/nix/profiles/system $path
+```
+you can now do
+```
+# nix copy --from ssh://server --profile /nix/var/nix/profiles/system $path
+```
+The advantage is that this avoids a time window where *path* is not a garbage collector root, and so could be deleted by a concurrent `nix store gc` process.

src/libcmd/command.cc

@@ -179,30 +179,34 @@ BuiltPathsCommand::BuiltPathsCommand(bool recursive)
 
 void BuiltPathsCommand::run(ref<Store> store, Installables && installables)
 {
-    BuiltPaths paths;
+    BuiltPaths rootPaths, allPaths;
+
     if (all) {
         if (installables.size())
             throw UsageError("'--all' does not expect arguments");
         // XXX: Only uses opaque paths, ignores all the realisations
         for (auto & p : store->queryAllValidPaths())
-            paths.emplace_back(BuiltPath::Opaque{p});
+            rootPaths.emplace_back(BuiltPath::Opaque{p});
+        allPaths = rootPaths;
     } else {
-        paths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables);
+        rootPaths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables);
+        allPaths = rootPaths;
+
         if (recursive) {
             // XXX: This only computes the store path closure, ignoring
             // intermediate realisations
             StorePathSet pathsRoots, pathsClosure;
-            for (auto & root : paths) {
+            for (auto & root : rootPaths) {
                 auto rootFromThis = root.outPaths();
                 pathsRoots.insert(rootFromThis.begin(), rootFromThis.end());
             }
             store->computeFSClosure(pathsRoots, pathsClosure);
             for (auto & path : pathsClosure)
-                paths.emplace_back(BuiltPath::Opaque{path});
+                allPaths.emplace_back(BuiltPath::Opaque{path});
         }
     }
 
-    run(store, std::move(paths));
+    run(store, std::move(allPaths), std::move(rootPaths));
 }
 
 StorePathsCommand::StorePathsCommand(bool recursive)
@@ -210,10 +214,10 @@ StorePathsCommand::StorePathsCommand(bool recursive)
 {
 }
 
-void StorePathsCommand::run(ref<Store> store, BuiltPaths && paths)
+void StorePathsCommand::run(ref<Store> store, BuiltPaths && allPaths, BuiltPaths && rootPaths)
 {
     StorePathSet storePaths;
-    for (auto & builtPath : paths)
+    for (auto & builtPath : allPaths)
         for (auto & p : builtPath.outPaths())
             storePaths.insert(p);
 
@@ -245,7 +249,7 @@ void MixProfile::updateProfile(const StorePath & storePath)
 {
     if (!profile)
         return;
-    auto store = getStore().dynamic_pointer_cast<LocalFSStore>();
+    auto store = getDstStore().dynamic_pointer_cast<LocalFSStore>();
     if (!store)
         throw Error("'--profile' is not supported for this Nix store");
     auto profile2 = absPath(*profile);
@@ -365,4 +369,31 @@ void MixEnvironment::setEnviron()
     return;
 }
 
+void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & buildables, LocalFSStore & store)
+{
+    for (const auto & [_i, buildable] : enumerate(buildables)) {
+        auto i = _i;
+        std::visit(
+            overloaded{
+                [&](const BuiltPath::Opaque & bo) {
+                    auto symlink = outLink;
+                    if (i)
+                        symlink += fmt("-%d", i);
+                    store.addPermRoot(bo.path, absPath(symlink.string()));
+                },
+                [&](const BuiltPath::Built & bfd) {
+                    for (auto & output : bfd.outputs) {
+                        auto symlink = outLink;
+                        if (i)
+                            symlink += fmt("-%d", i);
+                        if (output.first != "out")
+                            symlink += fmt("-%s", output.first);
+                        store.addPermRoot(output.second, absPath(symlink.string()));
+                    }
+                },
+            },
+            buildable.raw());
+    }
+}
+
 }

src/libcmd/command.hh

@@ -18,6 +18,7 @@ extern char ** savedArgv;
 class EvalState;
 struct Pos;
 class Store;
+class LocalFSStore;
 
 static constexpr Command::Category catHelp = -1;
 static constexpr Command::Category catSecondary = 100;
@@ -46,7 +47,20 @@ struct StoreCommand : virtual Command
 {
     StoreCommand();
     void run() override;
+
+    /**
+     * Return the default Nix store.
+     */
     ref<Store> getStore();
+
+    /**
+     * Return the destination Nix store.
+     */
+    virtual ref<Store> getDstStore()
+    {
+        return getStore();
+    }
+
     virtual ref<Store> createStore();
     /**
      * Main entry point, with a `Store` provided
@@ -69,7 +83,7 @@ struct CopyCommand : virtual StoreCommand
 
     ref<Store> createStore() override;
 
-    ref<Store> getDstStore();
+    ref<Store> getDstStore() override;
 };
 
 /**
@@ -239,7 +253,7 @@ public:
 
     BuiltPathsCommand(bool recursive = false);
 
-    virtual void run(ref<Store> store, BuiltPaths && paths) = 0;
+    virtual void run(ref<Store> store, BuiltPaths && allPaths, BuiltPaths && rootPaths) = 0;
 
     void run(ref<Store> store, Installables && installables) override;
 
@@ -252,7 +266,7 @@ struct StorePathsCommand : public BuiltPathsCommand
 
     virtual void run(ref<Store> store, StorePaths && storePaths) = 0;
 
-    void run(ref<Store> store, BuiltPaths && paths) override;
+    void run(ref<Store> store, BuiltPaths && allPaths, BuiltPaths && rootPaths) override;
 };
 
 /**
@@ -354,4 +368,10 @@ std::string showVersions(const std::set<std::string> & versions);
 void printClosureDiff(
     ref<Store> store, const StorePath & beforePath, const StorePath & afterPath, std::string_view indent);
 
+/**
+ * Create symlinks prefixed by `outLink` to the store paths in
+ * `buildables`.
+ */
+void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & buildables, LocalFSStore & store);
+
 }

src/libcmd/installables.cc

@@ -918,4 +918,12 @@ void BuiltPathsCommand::applyDefaultInstallables(std::vector<std::string> & rawI
         rawInstallables.push_back(".");
 }
 
+BuiltPaths toBuiltPaths(const std::vector<BuiltPathWithResult> & builtPathsWithResult)
+{
+    BuiltPaths res;
+    for (auto & i : builtPathsWithResult)
+        res.push_back(i.path);
+    return res;
+}
+
 }

src/libcmd/installables.hh

@@ -86,6 +86,8 @@ struct BuiltPathWithResult
     std::optional<BuildResult> result;
 };
 
+BuiltPaths toBuiltPaths(const std::vector<BuiltPathWithResult> & builtPathsWithResult);
+
 /**
  * Shorthand, for less typing and helping us keep the choice of
  * collection in sync.

src/nix/build.cc

@@ -42,29 +42,6 @@ static nlohmann::json builtPathsWithResultToJSON(const std::vector<BuiltPathWith
     return res;
 }
 
-// TODO deduplicate with other code also setting such out links.
-static void createOutLinks(const std::filesystem::path& outLink, const std::vector<BuiltPathWithResult>& buildables, LocalFSStore& store2)
-{
-    for (const auto & [_i, buildable] : enumerate(buildables)) {
-        auto i = _i;
-        std::visit(overloaded {
-            [&](const BuiltPath::Opaque & bo) {
-                auto symlink = outLink;
-                if (i) symlink += fmt("-%d", i);
-                store2.addPermRoot(bo.path, absPath(symlink.string()));
-            },
-            [&](const BuiltPath::Built & bfd) {
-                for (auto & output : bfd.outputs) {
-                    auto symlink = outLink;
-                    if (i) symlink += fmt("-%d", i);
-                    if (output.first != "out") symlink += fmt("-%s", output.first);
-                    store2.addPermRoot(output.second, absPath(symlink.string()));
-                }
-            },
-        }, buildable.path.raw());
-    }
-}
-
 struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
 {
     Path outLink = "result";
@@ -140,7 +117,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
 
         if (outLink != "")
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
-                createOutLinks(outLink, buildables, *store2);
+                createOutLinks(outLink, toBuiltPaths(buildables), *store2);
 
         if (printOutputPaths) {
             stopProgressBar();

src/nix/copy.cc

@@ -1,11 +1,13 @@
 #include "command.hh"
 #include "shared.hh"
 #include "store-api.hh"
+#include "local-fs-store.hh"
 
 using namespace nix;
 
-struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand
+struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile
 {
+    std::optional<std::filesystem::path> outLink;
     CheckSigsFlag checkSigs = CheckSigs;
 
     SubstituteFlag substitute = NoSubstitute;
@@ -13,6 +15,15 @@ struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand
     CmdCopy()
         : BuiltPathsCommand(true)
     {
+        addFlag({
+            .longName = "out-link",
+            .shortName = 'o',
+            .description = "Create symlinks prefixed with *path* to the top-level store paths fetched from the source store.",
+            .labels = {"path"},
+            .handler = {&outLink},
+            .completer = completePath
+        });
+
         addFlag({
             .longName = "no-check-sigs",
             .description = "Do not require that paths are signed by trusted keys.",
@@ -43,19 +54,28 @@ struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand
 
     Category category() override { return catSecondary; }
 
-    void run(ref<Store> srcStore, BuiltPaths && paths) override
+    void run(ref<Store> srcStore, BuiltPaths && allPaths, BuiltPaths && rootPaths) override
     {
         auto dstStore = getDstStore();
 
         RealisedPath::Set stuffToCopy;
 
-        for (auto & builtPath : paths) {
+        for (auto & builtPath : allPaths) {
             auto theseRealisations = builtPath.toRealisedPaths(*srcStore);
             stuffToCopy.insert(theseRealisations.begin(), theseRealisations.end());
         }
 
         copyPaths(
             *srcStore, *dstStore, stuffToCopy, NoRepair, checkSigs, substitute);
+
+        updateProfile(rootPaths);
+
+        if (outLink) {
+            if (auto store2 = dstStore.dynamic_pointer_cast<LocalFSStore>())
+                createOutLinks(*outLink, rootPaths, *store2);
+            else
+                throw Error("'--out-link' is not supported for this Nix store");
+        }
     }
 };
 

src/nix/copy.md

@@ -55,6 +55,15 @@ R""(
   # nix copy --to /tmp/nix nixpkgs#hello --no-check-sigs
   ```
 
+* Update the NixOS system profile to point to a closure copied from a
+  remote machine:
+
+  ```console
+  # nix copy --from ssh://server \
+      --profile /nix/var/nix/profiles/system \
+      /nix/store/r14v3km89zm3prwsa521fab5kgzvfbw4-nixos-system-foobar-24.05.20240925.759537f
+  ```
+
 # Description
 
 `nix copy` copies store path closures between two Nix stores. The

src/nix/realisation.cc

@@ -36,7 +36,7 @@ struct CmdRealisationInfo : BuiltPathsCommand, MixJSON
 
     Category category() override { return catSecondary; }
 
-    void run(ref<Store> store, BuiltPaths && paths) override
+    void run(ref<Store> store, BuiltPaths && paths, BuiltPaths && rootPaths) override
     {
         experimentalFeatureSettings.require(Xp::CaDerivations);
         RealisedPath::Set realisations;

tests/functional/zstd.sh

@@ -18,7 +18,10 @@ HASH=$(nix hash path "$outPath")
 clearStore
 clearCacheCache
 
-nix copy --from "$cacheURI" "$outPath" --no-check-sigs
+nix copy --from "$cacheURI" "$outPath" --no-check-sigs --profile "$TEST_ROOT/profile" --out-link "$TEST_ROOT/result"
+
+[[ -e $TEST_ROOT/profile ]]
+[[ -e $TEST_ROOT/result ]]
 
 if ls "$cacheDir/nar/"*.zst &> /dev/null; then
     echo "files do exist"