addMultipleToStore(): Move pathsToCopy

This allows RemoteStore::addMultipleToStore() to free the Source
objects early (and in particular the associated sinkToSource()
buffers). This should fix #7359. For example, memory consumption of

  nix copy --derivation --to ssh-ng://localhost?remote-store=/tmp/nix --derivation --no-check-sigs \
    /nix/store/4p9xmfgnvclqpii8pxqcwcvl9bxqy2xf-nixos-system-...drv

went from 353 MB to 74 MB.
This commit is contained in:
Eelco Dolstra 2025-01-20 14:23:02 +01:00
parent bcb92a5fa0
commit cc838e8181
4 changed files with 9 additions and 6 deletions

View File

@ -534,14 +534,16 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
void RemoteStore::addMultipleToStore(
PathsSource & pathsToCopy,
PathsSource && pathsToCopy,
Activity & act,
RepairFlag repair,
CheckSigsFlag checkSigs)
{
auto source = sinkToSource([&](Sink & sink) {
sink << pathsToCopy.size();
for (auto & [pathInfo, pathSource] : pathsToCopy) {
std::reverse(pathsToCopy.begin(), pathsToCopy.end());
while (!pathsToCopy.empty()) {
auto & [pathInfo, pathSource] = pathsToCopy.back();
WorkerProto::Serialise<ValidPathInfo>::write(*this,
WorkerProto::WriteConn {
.to = sink,
@ -549,6 +551,7 @@ void RemoteStore::addMultipleToStore(
},
pathInfo);
pathSource->drainInto(sink);
pathsToCopy.pop_back();
}
});

View File

@ -102,7 +102,7 @@ public:
CheckSigsFlag checkSigs) override;
void addMultipleToStore(
PathsSource & pathsToCopy,
PathsSource && pathsToCopy,
Activity & act,
RepairFlag repair,
CheckSigsFlag checkSigs) override;

View File

@ -223,7 +223,7 @@ StorePath Store::addToStore(
}
void Store::addMultipleToStore(
PathsSource & pathsToCopy,
PathsSource && pathsToCopy,
Activity & act,
RepairFlag repair,
CheckSigsFlag checkSigs)
@ -1138,7 +1138,7 @@ std::map<StorePath, StorePath> copyPaths(
pathsToCopy.push_back(std::pair{infoForDst, std::move(source)});
}
dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs);
dstStore.addMultipleToStore(std::move(pathsToCopy), act, repair, checkSigs);
return pathsMap;
}

View File

@ -425,7 +425,7 @@ public:
CheckSigsFlag checkSigs = CheckSigs);
virtual void addMultipleToStore(
PathsSource & pathsToCopy,
PathsSource && pathsToCopy,
Activity & act,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs);