Revert "Revert "Adapt scheduler to work with dynamic derivations""

This fixes dynamic derivations, reverting #9081. #9052 However will be
reintroduced unless this is modified somehow.

This reverts commit 8440afbed7.
This commit is contained in:
John Ericson 2023-11-20 16:38:41 -05:00
parent c6b5503190
commit 625ed1c0af
12 changed files with 390 additions and 48 deletions

View File

@ -0,0 +1,157 @@
#include "create-derivation-and-realise-goal.hh"
#include "worker.hh"
namespace nix {
CreateDerivationAndRealiseGoal::CreateDerivationAndRealiseGoal(ref<SingleDerivedPath> drvReq,
const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
: Goal(worker, DerivedPath::Built { .drvPath = drvReq, .outputs = wantedOutputs })
, drvReq(drvReq)
, wantedOutputs(wantedOutputs)
, buildMode(buildMode)
{
state = &CreateDerivationAndRealiseGoal::getDerivation;
name = fmt(
"outer obtaining drv from '%s' and then building outputs %s",
drvReq->to_string(worker.store),
std::visit(overloaded {
[&](const OutputsSpec::All) -> std::string {
return "* (all of them)";
},
[&](const OutputsSpec::Names os) {
return concatStringsSep(", ", quoteStrings(os));
},
}, wantedOutputs.raw));
trace("created outer");
worker.updateProgress();
}
CreateDerivationAndRealiseGoal::~CreateDerivationAndRealiseGoal()
{
}
static StorePath pathPartOfReq(const SingleDerivedPath & req)
{
return std::visit(overloaded {
[&](const SingleDerivedPath::Opaque & bo) {
return bo.path;
},
[&](const SingleDerivedPath::Built & bfd) {
return pathPartOfReq(*bfd.drvPath);
},
}, req.raw());
}
std::string CreateDerivationAndRealiseGoal::key()
{
/* Ensure that derivations get built in order of their name,
i.e. a derivation named "aardvark" always comes before "baboon". And
substitution goals and inner derivation goals always happen before
derivation goals (due to "b$"). */
return "c$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + drvReq->to_string(worker.store);
}
void CreateDerivationAndRealiseGoal::timedOut(Error && ex)
{
}
void CreateDerivationAndRealiseGoal::work()
{
(this->*state)();
}
void CreateDerivationAndRealiseGoal::addWantedOutputs(const OutputsSpec & outputs)
{
/* If we already want all outputs, there is nothing to do. */
auto newWanted = wantedOutputs.union_(outputs);
bool needRestart = !newWanted.isSubsetOf(wantedOutputs);
wantedOutputs = newWanted;
if (!needRestart) return;
if (!optDrvPath)
// haven't started steps where the outputs matter yet
return;
worker.makeDerivationGoal(*optDrvPath, outputs, buildMode);
}
void CreateDerivationAndRealiseGoal::getDerivation()
{
trace("outer init");
/* The first thing to do is to make sure that the derivation
exists. If it doesn't, it may be created through a
substitute. */
if (auto optDrvPath = [this]() -> std::optional<StorePath> {
if (buildMode != bmNormal) return std::nullopt;
auto drvPath = StorePath::dummy;
try {
drvPath = resolveDerivedPath(worker.store, *drvReq);
} catch (MissingRealisation &) {
return std::nullopt;
}
return worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath)
? std::optional { drvPath }
: std::nullopt;
}()) {
trace(fmt("already have drv '%s' for '%s', can go straight to building",
worker.store.printStorePath(*optDrvPath),
drvReq->to_string(worker.store)));
loadAndBuildDerivation();
} else {
trace("need to obtain drv we want to build");
addWaitee(worker.makeGoal(DerivedPath::fromSingle(*drvReq)));
state = &CreateDerivationAndRealiseGoal::loadAndBuildDerivation;
if (waitees.empty()) work();
}
}
void CreateDerivationAndRealiseGoal::loadAndBuildDerivation()
{
trace("outer load and build derivation");
if (nrFailed != 0) {
amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store)));
return;
}
StorePath drvPath = resolveDerivedPath(worker.store, *drvReq);
/* Build this step! */
concreteDrvGoal = worker.makeDerivationGoal(drvPath, wantedOutputs, buildMode);
addWaitee(upcast_goal(concreteDrvGoal));
state = &CreateDerivationAndRealiseGoal::buildDone;
optDrvPath = std::move(drvPath);
if (waitees.empty()) work();
}
void CreateDerivationAndRealiseGoal::buildDone()
{
trace("outer build done");
buildResult = upcast_goal(concreteDrvGoal)->getBuildResult(DerivedPath::Built {
.drvPath = drvReq,
.outputs = wantedOutputs,
});
if (buildResult.success())
amDone(ecSuccess);
else
amDone(ecFailed, Error("building '%s' failed", drvReq->to_string(worker.store)));
}
}

View File

@ -0,0 +1,96 @@
#pragma once
#include "parsed-derivations.hh"
#include "lock.hh"
#include "store-api.hh"
#include "pathlocks.hh"
#include "goal.hh"
namespace nix {
struct DerivationGoal;
/**
* This goal type is essentially the serial composition (like function
* composition) of a goal for getting a derivation, and then a
* `DerivationGoal` using the newly-obtained derivation.
*
* In the (currently experimental) general inductive case of derivations
* that are themselves build outputs, that first goal will be *another*
* `CreateDerivationAndRealiseGoal`. In the (much more common) base-case
* where the derivation has no provence and is just referred to by
* (content-addressed) store path, that first goal is a
* `SubstitutionGoal`.
*
* If we already have the derivation (e.g. if the evalutator has created
* the derivation locally and then instructured the store to build it),
* we can skip the first goal entirely as a small optimization.
*/
struct CreateDerivationAndRealiseGoal : public Goal
{
/**
* How to obtain a store path of the derivation to build.
*/
ref<SingleDerivedPath> drvReq;
/**
* The path of the derivation, once obtained.
**/
std::optional<StorePath> optDrvPath;
/**
* The goal for the corresponding concrete derivation.
**/
std::shared_ptr<DerivationGoal> concreteDrvGoal;
/**
* The specific outputs that we need to build.
*/
OutputsSpec wantedOutputs;
typedef void (CreateDerivationAndRealiseGoal::*GoalState)();
GoalState state;
/**
* The final output paths of the build.
*
* - For input-addressed derivations, always the precomputed paths
*
* - For content-addressed derivations, calcuated from whatever the
* hash ends up being. (Note that fixed outputs derivations that
* produce the "wrong" output still install that data under its
* true content-address.)
*/
OutputPathMap finalOutputs;
BuildMode buildMode;
CreateDerivationAndRealiseGoal(ref<SingleDerivedPath> drvReq,
const OutputsSpec & wantedOutputs, Worker & worker,
BuildMode buildMode = bmNormal);
virtual ~CreateDerivationAndRealiseGoal();
void timedOut(Error && ex) override;
std::string key() override;
void work() override;
/**
* Add wanted outputs to an already existing derivation goal.
*/
void addWantedOutputs(const OutputsSpec & outputs);
/**
* The states.
*/
void getDerivation();
void loadAndBuildDerivation();
void buildDone();
JobCategory jobCategory() const override {
return JobCategory::Administration;
};
};
}

View File

@ -44,7 +44,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
, wantedOutputs(wantedOutputs)
, buildMode(buildMode)
{
state = &DerivationGoal::getDerivation;
state = &DerivationGoal::loadDerivation;
name = fmt(
"building of '%s' from .drv file",
DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store));
@ -139,24 +139,6 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
}
void DerivationGoal::getDerivation()
{
trace("init");
/* The first thing to do is to make sure that the derivation
exists. If it doesn't, it may be created through a
substitute. */
if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) {
loadDerivation();
return;
}
addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath)));
state = &DerivationGoal::loadDerivation;
}
void DerivationGoal::loadDerivation()
{
trace("loading derivation");
@ -1579,23 +1561,24 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
if (!useDerivation) return;
auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
auto * dg = dynamic_cast<DerivationGoal *>(&*waitee);
if (!dg) return;
std::optional info = tryGetConcreteDrvGoal(waitee);
if (!info) return;
const auto & [dg, drvReq] = *info;
auto * nodeP = fullDrv.inputDrvs.findSlot(DerivedPath::Opaque { .path = dg->drvPath });
auto * nodeP = fullDrv.inputDrvs.findSlot(drvReq.get());
if (!nodeP) return;
auto & outputs = nodeP->value;
for (auto & outputName : outputs) {
auto buildResult = dg->getBuildResult(DerivedPath::Built {
.drvPath = makeConstantStorePathRef(dg->drvPath),
auto buildResult = dg.get().getBuildResult(DerivedPath::Built {
.drvPath = makeConstantStorePathRef(dg.get().drvPath),
.outputs = OutputsSpec::Names { outputName },
});
if (buildResult.success()) {
auto i = buildResult.builtOutputs.find(outputName);
if (i != buildResult.builtOutputs.end())
inputDrvOutputs.insert_or_assign(
{ dg->drvPath, outputName },
{ dg.get().drvPath, outputName },
i->second.outPath);
}
}

View File

@ -56,6 +56,10 @@ struct InitialOutput {
/**
* A goal for building some or all of the outputs of a derivation.
*
* The derivation must already be present, either in the store in a drv
* or in memory. If the derivation itself needs to be gotten first, a
* `CreateDerivationAndRealiseGoal` goal must be used instead.
*/
struct DerivationGoal : public Goal
{
@ -237,7 +241,6 @@ struct DerivationGoal : public Goal
/**
* The states.
*/
void getDerivation();
void loadDerivation();
void haveDerivation();
void outputsSubstitutionTried();

View File

@ -1,6 +1,7 @@
#include "worker.hh"
#include "substitution-goal.hh"
#ifndef _WIN32 // TODO Enable building on Windows
# include "create-derivation-and-realise-goal.hh"
# include "derivation-goal.hh"
#endif
#include "local-store.hh"
@ -29,8 +30,8 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
}
if (i->exitCode != Goal::ecSuccess) {
#ifndef _WIN32 // TODO Enable building on Windows
if (auto i2 = dynamic_cast<DerivationGoal *>(i.get()))
failed.insert(printStorePath(i2->drvPath));
if (auto i2 = dynamic_cast<CreateDerivationAndRealiseGoal *>(i.get()))
failed.insert(i2->drvReq->to_string(*this));
else
#endif
if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))

View File

@ -49,6 +49,16 @@ enum struct JobCategory {
* A substitution an arbitrary store object; it will use network resources.
*/
Substitution,
/**
* A goal that does no "real" work by itself, and just exists to depend on
* other goals which *do* do real work. These goals therefore are not
* limited.
*
* These goals cannot infinitely create themselves, so there is no risk of
* a "fork bomb" type situation (which would be a problem even though the
* goal do no real work) either.
*/
Administration,
};
struct Goal : public std::enable_shared_from_this<Goal>

View File

@ -5,6 +5,7 @@
#include "drv-output-substitution-goal.hh"
#include "derivation-goal.hh"
#ifndef _WIN32 // TODO Enable building on Windows
# include "create-derivation-and-realise-goal.hh"
# include "local-derivation-goal.hh"
# include "hook-instance.hh"
#endif
@ -43,6 +44,24 @@ Worker::~Worker()
}
std::shared_ptr<CreateDerivationAndRealiseGoal> Worker::makeCreateDerivationAndRealiseGoal(
ref<SingleDerivedPath> drvReq,
const OutputsSpec & wantedOutputs,
BuildMode buildMode)
{
std::weak_ptr<CreateDerivationAndRealiseGoal> & goal_weak = outerDerivationGoals.ensureSlot(*drvReq).value;
std::shared_ptr<CreateDerivationAndRealiseGoal> goal = goal_weak.lock();
if (!goal) {
goal = std::make_shared<CreateDerivationAndRealiseGoal>(drvReq, wantedOutputs, *this, buildMode);
goal_weak = goal;
wakeUp(goal);
} else {
goal->addWantedOutputs(wantedOutputs);
}
return goal;
}
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
const StorePath & drvPath,
const OutputsSpec & wantedOutputs,
@ -120,10 +139,7 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
{
return std::visit(overloaded {
[&](const DerivedPath::Built & bfd) -> GoalPtr {
if (auto bop = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath))
return makeDerivationGoal(bop->path, bfd.outputs, buildMode);
else
throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented.");
return makeCreateDerivationAndRealiseGoal(bfd.drvPath, bfd.outputs, buildMode);
},
[&](const DerivedPath::Opaque & bo) -> GoalPtr {
return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
@ -132,24 +148,46 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
}
template<typename K, typename V, typename F>
static void cullMap(std::map<K, V> & goalMap, F f)
{
for (auto i = goalMap.begin(); i != goalMap.end();)
if (!f(i->second))
i = goalMap.erase(i);
else ++i;
}
template<typename K, typename G>
static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
{
/* !!! inefficient */
for (auto i = goalMap.begin();
i != goalMap.end(); )
if (i->second.lock() == goal) {
auto j = i; ++j;
goalMap.erase(i);
i = j;
}
else ++i;
cullMap(goalMap, [&](const std::weak_ptr<G> & gp) -> bool {
return gp.lock() != goal;
});
}
template<typename K>
static void removeGoal(std::shared_ptr<CreateDerivationAndRealiseGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode> & goalMap);
template<typename K>
static void removeGoal(std::shared_ptr<CreateDerivationAndRealiseGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode> & goalMap)
{
/* !!! inefficient */
cullMap(goalMap, [&](DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode & node) -> bool {
if (node.value.lock() == goal)
node.value.reset();
removeGoal(goal, node.childMap);
return !node.value.expired() || !node.childMap.empty();
});
}
void Worker::removeGoal(GoalPtr goal)
{
if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
if (auto drvGoal = std::dynamic_pointer_cast<CreateDerivationAndRealiseGoal>(goal))
nix::removeGoal(drvGoal, outerDerivationGoals.map);
else if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
nix::removeGoal(drvGoal, derivationGoals);
else
if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
@ -215,6 +253,9 @@ void Worker::childStarted(GoalPtr goal, const std::set<MuxablePipePollState::Com
case JobCategory::Build:
nrLocalBuilds++;
break;
case JobCategory::Administration:
/* Intentionally not limited, see docs */
break;
default:
abort();
}
@ -238,6 +279,9 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
assert(nrLocalBuilds > 0);
nrLocalBuilds--;
break;
case JobCategory::Administration:
/* Intentionally not limited, see docs */
break;
default:
abort();
}
@ -290,9 +334,9 @@ void Worker::run(const Goals & _topGoals)
for (auto & i : _topGoals) {
topGoals.insert(i);
if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
if (auto goal = dynamic_cast<CreateDerivationAndRealiseGoal *>(i.get())) {
topPaths.push_back(DerivedPath::Built {
.drvPath = makeConstantStorePathRef(goal->drvPath),
.drvPath = goal->drvReq,
.outputs = goal->wantedOutputs,
});
} else
@ -556,4 +600,19 @@ GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal)
return subGoal;
}
GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal)
{
return subGoal;
}
std::optional<std::pair<std::reference_wrapper<const DerivationGoal>, std::reference_wrapper<const SingleDerivedPath>>> tryGetConcreteDrvGoal(GoalPtr waitee)
{
auto * odg = dynamic_cast<CreateDerivationAndRealiseGoal *>(&*waitee);
if (!odg) return std::nullopt;
return {{
std::cref(*odg->concreteDrvGoal),
std::cref(*odg->drvReq),
}};
}
}

View File

@ -3,6 +3,7 @@
#include "types.hh"
#include "store-api.hh"
#include "derived-path-map.hh"
#include "goal.hh"
#include "realisation.hh"
#include "muxable-pipe.hh"
@ -13,6 +14,7 @@
namespace nix {
/* Forward definition. */
struct CreateDerivationAndRealiseGoal;
struct DerivationGoal;
struct PathSubstitutionGoal;
class DrvOutputSubstitutionGoal;
@ -31,9 +33,25 @@ class DrvOutputSubstitutionGoal;
*/
GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal);
GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal);
typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
/**
* The current implementation of impure derivations has
* `DerivationGoal`s accumulate realisations from their waitees.
* Unfortunately, `DerivationGoal`s don't directly depend on other
* goals, but instead depend on `CreateDerivationAndRealiseGoal`s.
*
* We try not to share any of the details of any goal type with any
* other, for sake of modularity and quicker rebuilds. This means we
* cannot "just" downcast and fish out the field. So as an escape hatch,
* we have made the function, written in `worker.cc` where all the goal
* types are visible, and use it instead.
*/
std::optional<std::pair<std::reference_wrapper<const DerivationGoal>, std::reference_wrapper<const SingleDerivedPath>>> tryGetConcreteDrvGoal(GoalPtr waitee);
/**
* A mapping used to remember for each child process to what goal it
* belongs, and comm channels for receiving log data and output
@ -103,6 +121,9 @@ private:
* Maps used to prevent multiple instantiations of a goal for the
* same derivation / path.
*/
DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>> outerDerivationGoals;
std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
@ -196,6 +217,9 @@ public:
* @ref DerivationGoal "derivation goal"
*/
private:
std::shared_ptr<CreateDerivationAndRealiseGoal> makeCreateDerivationAndRealiseGoal(
ref<SingleDerivedPath> drvPath,
const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
const StorePath & drvPath, const OutputsSpec & wantedOutputs,
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);

View File

@ -52,6 +52,7 @@ typename DerivedPathMap<V>::ChildNode * DerivedPathMap<V>::findSlot(const Single
// instantiations
#include "create-derivation-and-realise-goal.hh"
namespace nix {
template<>
@ -68,4 +69,7 @@ std::strong_ordering DerivedPathMap<std::set<std::string>>::ChildNode::operator
template struct DerivedPathMap<std::set<std::string>>::ChildNode;
template struct DerivedPathMap<std::set<std::string>>;
template struct DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>;
};

View File

@ -21,8 +21,11 @@ namespace nix {
*
* @param V A type to instantiate for each output. It should probably
* should be an "optional" type so not every interior node has to have a
* value. `* const Something` or `std::optional<Something>` would be
* good choices for "optional" types.
* value. For example, the scheduler uses
* `DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>` to
* remember which goals correspond to which outputs. `* const Something`
* or `std::optional<Something>` would also be good choices for
* "optional" types.
*/
template<typename V>
struct DerivedPathMap {

View File

@ -18,4 +18,6 @@ clearStore
drvDep=$(nix-instantiate ./text-hashed-output.nix -A producingDrv)
expectStderr 1 nix build "${drvDep}^out^out" --no-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented"
out2=$(nix build "${drvDep}^out^out" --no-link)
test $out1 == $out2

View File

@ -6,6 +6,6 @@ out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link)
clearStore
expectStderr 1 nix-build ./text-hashed-output.nix -A wrapper --no-out-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented"
out2=$(nix-build ./text-hashed-output.nix -A wrapper --no-out-link)
# diff -r $out1 $out2
diff -r $out1 $out2