nordic-dev.net/nix
nordic-dev.net/nix
2
0
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2024-11-22 14:52:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cache the Nix evaluation when possible

Browse Source 
Hook into the evaluation to cache to cache the evaluation of flake
members (stuff like `(builtins.getCache foo).bar.baz`).
Also replaces the old eval cache that was used at the cli level.
This commit is contained in:
regnat 2021-02-01 17:20:17 +01:00
parent 4e98f0345c
commit 188ed75d54
27 changed files with 1324 additions and 1089 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

157
src/libcmd/installables.cc
View File

@ -9,7 +9,6 @@
#include "store-api.hh"
#include "shared.hh"
#include "flake/flake.hh"
#include "eval-cache.hh"
#include "url.hh"
#include "registry.hh"
@ -222,10 +221,8 @@ void completeFlakeRefWithFragment(
// FIXME: do tilde expansion.
auto flakeRef = parseFlakeRef(flakeRefS, absPath("."));
auto evalCache = openEvalCache(*evalState,
std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags)));
auto root = evalCache->getRoot();
auto lockedFlake = lockFlake(*evalState, flakeRef, lockFlags);
auto rootValue = getFlakeValue(*evalState, lockedFlake);
/* Complete 'fragment' relative to all the
attrpath prefixes as well as the root of the
@ -243,12 +240,18 @@ void completeFlakeRefWithFragment(
attrPath.pop_back();
}
auto attr = root->findAlongAttrPath(attrPath);
if (!attr) continue;
auto cachedFieldNames = rootValue->getCache().listChildrenAtPath(evalState->symbols, attrPath);
for (auto & attr2 : attr->getAttrs()) {
if (hasPrefix(attr2, lastAttr)) {
auto attrPath2 = attr->getAttrPath(attr2);
if (!cachedFieldNames) {
auto accessResult = evalState->getOptionalAttrField(*rootValue, attrPath, noPos);
if (accessResult.error) continue;
cachedFieldNames = evalState->listAttrFields(*accessResult.value, *accessResult.pos);
}
for (auto & lastFieldName : *cachedFieldNames) {
if (hasPrefix(lastFieldName, lastAttr)) {
auto attrPath2 = attrPath;
attrPath2.push_back(lastFieldName);
/* Strip the attrpath prefix. */
attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size());
completions->add(flakeRefS + "#" + concatStringsSep(".", attrPath2));
@ -260,8 +263,8 @@ void completeFlakeRefWithFragment(
attrpaths. */
if (fragment.empty()) {
for (auto & attrPath : defaultFlakeAttrPaths) {
auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath));
if (!attr) continue;
auto accessResult = evalState->getOptionalAttrField(*rootValue, {evalState->symbols.create(attrPath)}, noPos);
if (accessResult.error) continue;
completions->add(flakeRefS + "#");
}
}
@ -311,24 +314,6 @@ Buildable Installable::toBuildable()
return std::move(buildables[0]);
}
std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
Installable::getCursors(EvalState & state)
{
auto evalCache =
std::make_shared<nix::eval_cache::EvalCache>(std::nullopt, state,
[&]() { return toValue(state).first; });
return {{evalCache->getRoot(), ""}};
}
std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
Installable::getCursor(EvalState & state)
{
auto cursors = getCursors(state);
if (cursors.empty())
throw Error("cannot find flake attribute '%s'", what());
return cursors[0];
}
struct InstallableStorePath : Installable
{
ref<Store> store;
@ -399,11 +384,11 @@ struct InstallableAttrPath : InstallableValue
std::string what() override { return attrPath; }
std::pair<Value *, Pos> toValue(EvalState & state) override
std::vector<Installable::ValueInfo> toValues(EvalState & state) override
{
auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v);
state.forceValue(*vRes);
return {vRes, pos};
return {{vRes, pos, attrPath}};
}
virtual std::vector<InstallableValue::DerivationInfo> toDerivations() override;
@ -411,7 +396,7 @@ struct InstallableAttrPath : InstallableValue
std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations()
{
auto v = toValue(*state).first;
auto v = toValue(*state).value;
Bindings & autoArgs = *cmd.getAutoArgs(*state);
@ -445,45 +430,7 @@ std::vector<std::string> InstallableFlake::getActualAttrPaths()
Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake)
{
auto vFlake = state.allocValue();
callFlake(state, lockedFlake, *vFlake);
auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
assert(aOutputs);
state.forceValue(*aOutputs->value);
return aOutputs->value;
}
ref<eval_cache::EvalCache> openEvalCache(
EvalState & state,
std::shared_ptr<flake::LockedFlake> lockedFlake)
{
auto fingerprint = lockedFlake->getFingerprint();
return make_ref<nix::eval_cache::EvalCache>(
evalSettings.useEvalCache && evalSettings.pureEval
? std::optional { std::cref(fingerprint) }
: std::nullopt,
state,
[&state, lockedFlake]()
{
/* For testing whether the evaluation cache is
complete. */
if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0")
throw Error("not everything is cached, but evaluation is not allowed");
auto vFlake = state.allocValue();
flake::callFlake(state, *lockedFlake, *vFlake);
state.forceAttrs(*vFlake);
auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
assert(aOutputs);
return aOutputs->value;
});
return getFlakeValue(state, lockedFlake);
}
static std::string showAttrPaths(const std::vector<std::string> & paths)
@ -500,29 +447,20 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
{
auto lockedFlake = getLockedFlake();
auto cache = openEvalCache(*state, lockedFlake);
auto root = cache->getRoot();
auto flakeValue = toValue(*state);
for (auto & attrPath : getActualAttrPaths()) {
auto attr = root->findAlongAttrPath(parseAttrPath(*state, attrPath));
if (!attr) continue;
auto rawDrvInfo = getDerivation(*state, *flakeValue.value, false);
if (!rawDrvInfo)
throw Error("flake output attribute '%s' is not a derivation", flakeValue.positionInfo);
if (!attr->isDerivation())
throw Error("flake output attribute '%s' is not a derivation", attrPath);
auto drvInfo = InstallableValue::DerivationInfo
{
state->store->parseStorePath(rawDrvInfo->queryDrvPath()),
state->store->maybeParseStorePath(rawDrvInfo->queryOutPath()),
rawDrvInfo->queryOutputName()
};
auto drvPath = attr->forceDerivation();
auto drvInfo = DerivationInfo{
std::move(drvPath),
state->store->maybeParseStorePath(attr->getAttr(state->sOutPath)->getString()),
attr->getAttr(state->sOutputName)->getString()
};
return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)};
}
throw Error("flake '%s' does not provide attribute %s",
flakeRef, showAttrPaths(getActualAttrPaths()));
return {flakeValue.positionInfo, lockedFlake->flake.lockedRef, std::move(drvInfo)};
}
std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations()
@ -532,8 +470,20 @@ std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations()
return res;
}
std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state)
Installable::ValueInfo InstallableFlake::toValue(EvalState & state)
{
auto values = toValues(state);
if (values.empty())
throw Error("flake '%s' does not provide attribute %s",
flakeRef, showAttrPaths(getActualAttrPaths()));
return values[0];
}
std::vector<Installable::ValueInfo>
InstallableFlake::toValues(EvalState & state)
{
std::vector<Installable::ValueInfo> res;
auto lockedFlake = getLockedFlake();
auto vOutputs = getFlakeOutputs(state, *lockedFlake);
@ -544,30 +494,11 @@ std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state)
try {
auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs);
state.forceValue(*v);
return {v, pos};
res.push_back({v, pos, attrPath});
} catch (AttrPathNotFound & e) {
}
}
throw Error("flake '%s' does not provide attribute %s",
flakeRef, showAttrPaths(getActualAttrPaths()));
}
std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
InstallableFlake::getCursors(EvalState & state)
{
auto evalCache = openEvalCache(state,
std::make_shared<flake::LockedFlake>(lockFlake(state, flakeRef, lockFlags)));
auto root = evalCache->getRoot();
std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> res;
for (auto & attrPath : getActualAttrPaths()) {
auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath));
if (attr) res.push_back({attr, attrPath});
}
return res;
}

33
src/libcmd/installables.hh
View File

@ -54,7 +54,23 @@ struct Installable
App toApp(EvalState & state);
virtual std::pair<Value *, Pos> toValue(EvalState & state)
struct ValueInfo {
Value * value;
Pos pos;
string positionInfo;
};
virtual ValueInfo toValue(EvalState & state)
{
auto values = toValues(state);
if (values.empty())
throw Error("Installable '%s' does not provide a default value",
what());
return values[0];
}
virtual std::vector<ValueInfo>
toValues(EvalState & state)
{
throw Error("argument '%s' cannot be evaluated", what());
}
@ -66,11 +82,6 @@ struct Installable
return {};
}
virtual std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
getCursors(EvalState & state);
std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
getCursor(EvalState & state);
virtual FlakeRef nixpkgsFlakeRef() const
{
@ -120,18 +131,14 @@ struct InstallableFlake : InstallableValue
std::vector<DerivationInfo> toDerivations() override;
std::pair<Value *, Pos> toValue(EvalState & state) override;
ValueInfo toValue(EvalState & state) override;
std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
getCursors(EvalState & state) override;
std::vector<ValueInfo>
toValues(EvalState & state) override;
std::shared_ptr<flake::LockedFlake> getLockedFlake() const;
FlakeRef nixpkgsFlakeRef() const override;
};
ref<eval_cache::EvalCache> openEvalCache(
EvalState & state,
std::shared_ptr<flake::LockedFlake> lockedFlake);
}

9
src/libexpr/attr-path.cc
View File

@ -73,11 +73,12 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
if (attr.empty())
throw Error("empty attribute name in selection path '%1%'", attrPath);
Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
if (a == v->attrs->end())
Value * vRes = state.allocValue();
auto getResult = state.getOptionalAttrField(*v, {state.symbols.create(attr)}, pos, *vRes);
if (getResult.error)
throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath);
v = &*a->value;
pos = *a->pos;
v = vRes;
pos = *getResult.pos;
}
else {

4
src/libexpr/attr-set.hh
View File

@ -38,6 +38,10 @@ public:
private:
size_t size_, capacity_;
public:
private:
Attr attrs[0];
Bindings(size_t capacity) : size_(0), capacity_(capacity) { }

21
src/libexpr/context.cc Normal file
View File

@ -0,0 +1,21 @@
#include "context.hh"
namespace nix {
/* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */
std::pair<string, string> decodeContext(std::string_view s)
{
if (s.at(0) == '!') {
size_t index = s.find("!", 1);
return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
} else
return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
}
std::string encodeContext(std::string_view name, std::string_view path)
{
return "!" + std::string(name) + "!" + std::string(path);
}
}

11
src/libexpr/context.hh Normal file
View File

@ -0,0 +1,11 @@
#include "util.hh"
namespace nix {
/* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */
std::pair<string, string> decodeContext(std::string_view s);
std::string encodeContext(std::string_view name, std::string_view path);
}

629
src/libexpr/eval-cache.cc
View File

@ -1,629 +0,0 @@
#include "eval-cache.hh"
#include "sqlite.hh"
#include "eval.hh"
#include "eval-inline.hh"
#include "store-api.hh"
namespace nix::eval_cache {
static const char * schema = R"sql(
create table if not exists Attributes (
parent integer not null,
name text,
type integer not null,
value text,
context text,
primary key (parent, name)
);
)sql";
struct AttrDb
{
std::atomic_bool failed{false};
struct State
{
SQLite db;
SQLiteStmt insertAttribute;
SQLiteStmt insertAttributeWithContext;
SQLiteStmt queryAttribute;
SQLiteStmt queryAttributes;
std::unique_ptr<SQLiteTxn> txn;
};
std::unique_ptr<Sync<State>> _state;
AttrDb(const Hash & fingerprint)
: _state(std::make_unique<Sync<State>>())
{
auto state(_state->lock());
Path cacheDir = getCacheDir() + "/nix/eval-cache-v2";
createDirs(cacheDir);
Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
state->db = SQLite(dbPath);
state->db.isCache();
state->db.exec(schema);
state->insertAttribute.create(state->db,
"insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
state->insertAttributeWithContext.create(state->db,
"insert or replace into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)");
state->queryAttribute.create(state->db,
"select rowid, type, value, context from Attributes where parent = ? and name = ?");
state->queryAttributes.create(state->db,
"select name from Attributes where parent = ?");
state->txn = std::make_unique<SQLiteTxn>(state->db);
}
~AttrDb()
{
try {
auto state(_state->lock());
if (!failed)
state->txn->commit();
state->txn.reset();
} catch (...) {
ignoreException();
}
}
template<typename F>
AttrId doSQLite(F && fun)
{
if (failed) return 0;
try {
return fun();
} catch (SQLiteError &) {
ignoreException();
failed = true;
return 0;
}
}
AttrId setAttrs(
AttrKey key,
const std::vector<Symbol> & attrs)
{
return doSQLite([&]()
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::FullAttrs)
(0, false).exec();
AttrId rowId = state->db.getLastInsertedRowId();
assert(rowId);
for (auto & attr : attrs)
state->insertAttribute.use()
(rowId)
(attr)
(AttrType::Placeholder)
(0, false).exec();
return rowId;
});
}
AttrId setString(
AttrKey key,
std::string_view s,
const char * * context = nullptr)
{
return doSQLite([&]()
{
auto state(_state->lock());
if (context) {
std::string ctx;
for (const char * * p = context; *p; ++p) {
if (p != context) ctx.push_back(' ');
ctx.append(*p);
}
state->insertAttributeWithContext.use()
(key.first)
(key.second)
(AttrType::String)
(s)
(ctx).exec();
} else {
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::String)
(s).exec();
}
return state->db.getLastInsertedRowId();
});
}
AttrId setBool(
AttrKey key,
bool b)
{
return doSQLite([&]()
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Bool)
(b ? 1 : 0).exec();
return state->db.getLastInsertedRowId();
});
}
AttrId setPlaceholder(AttrKey key)
{
return doSQLite([&]()
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Placeholder)
(0, false).exec();
return state->db.getLastInsertedRowId();
});
}
AttrId setMissing(AttrKey key)
{
return doSQLite([&]()
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Missing)
(0, false).exec();
return state->db.getLastInsertedRowId();
});
}
AttrId setMisc(AttrKey key)
{
return doSQLite([&]()
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Misc)
(0, false).exec();
return state->db.getLastInsertedRowId();
});
}
AttrId setFailed(AttrKey key)
{
return doSQLite([&]()
{
auto state(_state->lock());
state->insertAttribute.use()
(key.first)
(key.second)
(AttrType::Failed)
(0, false).exec();
return state->db.getLastInsertedRowId();
});
}
std::optional<std::pair<AttrId, AttrValue>> getAttr(
AttrKey key,
SymbolTable & symbols)
{
auto state(_state->lock());
auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
if (!queryAttribute.next()) return {};
auto rowId = (AttrType) queryAttribute.getInt(0);
auto type = (AttrType) queryAttribute.getInt(1);
switch (type) {
case AttrType::Placeholder:
return {{rowId, placeholder_t()}};
case AttrType::FullAttrs: {
// FIXME: expensive, should separate this out.
std::vector<Symbol> attrs;
auto queryAttributes(state->queryAttributes.use()(rowId));
while (queryAttributes.next())
attrs.push_back(symbols.create(queryAttributes.getStr(0)));
return {{rowId, attrs}};
}
case AttrType::String: {
std::vector<std::pair<Path, std::string>> context;
if (!queryAttribute.isNull(3))
for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
context.push_back(decodeContext(s));
return {{rowId, string_t{queryAttribute.getStr(2), context}}};
}
case AttrType::Bool:
return {{rowId, queryAttribute.getInt(2) != 0}};
case AttrType::Missing:
return {{rowId, missing_t()}};
case AttrType::Misc:
return {{rowId, misc_t()}};
case AttrType::Failed:
return {{rowId, failed_t()}};
default:
throw Error("unexpected type in evaluation cache");
}
}
};
static std::shared_ptr<AttrDb> makeAttrDb(const Hash & fingerprint)
{
try {
return std::make_shared<AttrDb>(fingerprint);
} catch (SQLiteError &) {
ignoreException();
return nullptr;
}
}
EvalCache::EvalCache(
std::optional<std::reference_wrapper<const Hash>> useCache,
EvalState & state,
RootLoader rootLoader)
: db(useCache ? makeAttrDb(*useCache) : nullptr)
, state(state)
, rootLoader(rootLoader)
{
}
Value * EvalCache::getRootValue()
{
if (!value) {
debug("getting root value");
value = allocRootValue(rootLoader());
}
return *value;
}
std::shared_ptr<AttrCursor> EvalCache::getRoot()
{
return std::make_shared<AttrCursor>(ref(shared_from_this()), std::nullopt);
}
AttrCursor::AttrCursor(
ref<EvalCache> root,
Parent parent,
Value * value,
std::optional<std::pair<AttrId, AttrValue>> && cachedValue)
: root(root), parent(parent), cachedValue(std::move(cachedValue))
{
if (value)
_value = allocRootValue(value);
}
AttrKey AttrCursor::getKey()
{
if (!parent)
return {0, root->state.sEpsilon};
if (!parent->first->cachedValue) {
parent->first->cachedValue = root->db->getAttr(
parent->first->getKey(), root->state.symbols);
assert(parent->first->cachedValue);
}
return {parent->first->cachedValue->first, parent->second};
}
Value & AttrCursor::getValue()
{
if (!_value) {
if (parent) {
auto & vParent = parent->first->getValue();
root->state.forceAttrs(vParent);
auto attr = vParent.attrs->get(parent->second);
if (!attr)
throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr());
_value = allocRootValue(attr->value);
} else
_value = allocRootValue(root->getRootValue());
}
return **_value;
}
std::vector<Symbol> AttrCursor::getAttrPath() const
{
if (parent) {
auto attrPath = parent->first->getAttrPath();
attrPath.push_back(parent->second);
return attrPath;
} else
return {};
}
std::vector<Symbol> AttrCursor::getAttrPath(Symbol name) const
{
auto attrPath = getAttrPath();
attrPath.push_back(name);
return attrPath;
}
std::string AttrCursor::getAttrPathStr() const
{
return concatStringsSep(".", getAttrPath());
}
std::string AttrCursor::getAttrPathStr(Symbol name) const
{
return concatStringsSep(".", getAttrPath(name));
}
Value & AttrCursor::forceValue()
{
debug("evaluating uncached attribute %s", getAttrPathStr());
auto & v = getValue();
try {
root->state.forceValue(v);
} catch (EvalError &) {
debug("setting '%s' to failed", getAttrPathStr());
if (root->db)
cachedValue = {root->db->setFailed(getKey()), failed_t()};
throw;
}
if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
if (v.type() == nString)
cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context),
string_t{v.string.s, {}}};
else if (v.type() == nPath)
cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}};
else if (v.type() == nBool)
cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
else if (v.type() == nAttrs)
; // FIXME: do something?
else
cachedValue = {root->db->setMisc(getKey()), misc_t()};
}
return v;
}
std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors)
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
if (cachedValue) {
if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
for (auto & attr : *attrs)
if (attr == name)
return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name));
return nullptr;
} else if (std::get_if<placeholder_t>(&cachedValue->second)) {
auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols);
if (attr) {
if (std::get_if<missing_t>(&attr->second))
return nullptr;
else if (std::get_if<failed_t>(&attr->second)) {
if (forceErrors)
debug("reevaluating failed cached attribute '%s'");
else
throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name));
} else
return std::make_shared<AttrCursor>(root,
std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
}
// Incomplete attrset, so need to fall thru and
// evaluate to see whether 'name' exists
} else
return nullptr;
//throw TypeError("'%s' is not an attribute set", getAttrPathStr());
}
}
auto & v = forceValue();
if (v.type() != nAttrs)
return nullptr;
//throw TypeError("'%s' is not an attribute set", getAttrPathStr());
auto attr = v.attrs->get(name);
if (!attr) {
if (root->db) {
if (!cachedValue)
cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
root->db->setMissing({cachedValue->first, name});
}
return nullptr;
}
std::optional<std::pair<AttrId, AttrValue>> cachedValue2;
if (root->db) {
if (!cachedValue)
cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()};
}
return std::make_shared<AttrCursor>(
root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2));
}
std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
{
return maybeGetAttr(root->state.symbols.create(name));
}
std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors)
{
auto p = maybeGetAttr(name, forceErrors);
if (!p)
throw Error("attribute '%s' does not exist", getAttrPathStr(name));
return p;
}
std::shared_ptr<AttrCursor> AttrCursor::getAttr(std::string_view name)
{
return getAttr(root->state.symbols.create(name));
}
std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
{
auto res = shared_from_this();
for (auto & attr : attrPath) {
res = res->maybeGetAttr(attr);
if (!res) return {};
}
return res;
}
std::string AttrCursor::getString()
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
if (auto s = std::get_if<string_t>(&cachedValue->second)) {
debug("using cached string attribute '%s'", getAttrPathStr());
return s->first;
} else
throw TypeError("'%s' is not a string", getAttrPathStr());
}
}
auto & v = forceValue();
if (v.type() != nString && v.type() != nPath)
throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type()));
return v.type() == nString ? v.string.s : v.path;
}
string_t AttrCursor::getStringWithContext()
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
if (auto s = std::get_if<string_t>(&cachedValue->second)) {
bool valid = true;
for (auto & c : s->second) {
if (!root->state.store->isValidPath(root->state.store->parseStorePath(c.first))) {
valid = false;
break;
}
}
if (valid) {
debug("using cached string attribute '%s'", getAttrPathStr());
return *s;
}
} else
throw TypeError("'%s' is not a string", getAttrPathStr());
}
}
auto & v = forceValue();
if (v.type() == nString)
return {v.string.s, v.getContext()};
else if (v.type() == nPath)
return {v.path, {}};
else
throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type()));
}
bool AttrCursor::getBool()
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
if (auto b = std::get_if<bool>(&cachedValue->second)) {
debug("using cached Boolean attribute '%s'", getAttrPathStr());
return *b;
} else
throw TypeError("'%s' is not a Boolean", getAttrPathStr());
}
}
auto & v = forceValue();
if (v.type() != nBool)
throw TypeError("'%s' is not a Boolean", getAttrPathStr());
return v.boolean;
}
std::vector<Symbol> AttrCursor::getAttrs()
{
if (root->db) {
if (!cachedValue)
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
debug("using cached attrset attribute '%s'", getAttrPathStr());
return *attrs;
} else
throw TypeError("'%s' is not an attribute set", getAttrPathStr());
}
}
auto & v = forceValue();
if (v.type() != nAttrs)
throw TypeError("'%s' is not an attribute set", getAttrPathStr());
std::vector<Symbol> attrs;
for (auto & attr : *getValue().attrs)
attrs.push_back(attr.name);
std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) {
return (const string &) a < (const string &) b;
});
if (root->db)
cachedValue = {root->db->setAttrs(getKey(), attrs), attrs};
return attrs;
}
bool AttrCursor::isDerivation()
{
auto aType = maybeGetAttr("type");
return aType && aType->getString() == "derivation";
}
StorePath AttrCursor::forceDerivation()
{
auto aDrvPath = getAttr(root->state.sDrvPath, true);
auto drvPath = root->state.store->parseStorePath(aDrvPath->getString());
if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) {
/* The eval cache contains 'drvPath', but the actual path has
been garbage-collected. So force it to be regenerated. */
aDrvPath->forceValue();
if (!root->state.store->isValidPath(drvPath))
throw Error("don't know how to recreate store derivation '%s'!",
root->state.store->printStorePath(drvPath));
}
return drvPath;
}
}

123
src/libexpr/eval-cache.hh
View File

@ -1,123 +0,0 @@
#pragma once
#include "sync.hh"
#include "hash.hh"
#include "eval.hh"
#include <functional>
#include <variant>
namespace nix::eval_cache {
MakeError(CachedEvalError, EvalError);
struct AttrDb;
class AttrCursor;
class EvalCache : public std::enable_shared_from_this<EvalCache>
{
friend class AttrCursor;
std::shared_ptr<AttrDb> db;
EvalState & state;
typedef std::function<Value *()> RootLoader;
RootLoader rootLoader;
RootValue value;
Value * getRootValue();
public:
EvalCache(
std::optional<std::reference_wrapper<const Hash>> useCache,
EvalState & state,
RootLoader rootLoader);
std::shared_ptr<AttrCursor> getRoot();
};
enum AttrType {
Placeholder = 0,
FullAttrs = 1,
String = 2,
Missing = 3,
Misc = 4,
Failed = 5,
Bool = 6,
};
struct placeholder_t {};
struct missing_t {};
struct misc_t {};
struct failed_t {};
typedef uint64_t AttrId;
typedef std::pair<AttrId, Symbol> AttrKey;
typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t;
typedef std::variant<
std::vector<Symbol>,
string_t,
placeholder_t,
missing_t,
misc_t,
failed_t,
bool
> AttrValue;
class AttrCursor : public std::enable_shared_from_this<AttrCursor>
{
friend class EvalCache;
ref<EvalCache> root;
typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
Parent parent;
RootValue _value;
std::optional<std::pair<AttrId, AttrValue>> cachedValue;
AttrKey getKey();
Value & getValue();
public:
AttrCursor(
ref<EvalCache> root,
Parent parent,
Value * value = nullptr,
std::optional<std::pair<AttrId, AttrValue>> && cachedValue = {});
std::vector<Symbol> getAttrPath() const;
std::vector<Symbol> getAttrPath(Symbol name) const;
std::string getAttrPathStr() const;
std::string getAttrPathStr(Symbol name) const;
std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false);
std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
std::shared_ptr<AttrCursor> getAttr(Symbol name, bool forceErrors = false);
std::shared_ptr<AttrCursor> getAttr(std::string_view name);
std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath);
std::string getString();
string_t getStringWithContext();
bool getBool();
std::vector<Symbol> getAttrs();
bool isDerivation();
Value & forceValue();
/* Force creation of the .drv file in the Nix store. */
StorePath forceDerivation();
};
}

430
src/libexpr/eval.cc
View File

@ -433,6 +433,9 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
EvalState::~EvalState()
{
for (auto [_, cache] : evalCache) {
cache->commit();
}
}
@ -1118,48 +1121,352 @@ static string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPa
unsigned long nrLookups = 0;
tree_cache::AttrValue cachedValueFor(const Value& v)
{
tree_cache::AttrValue valueToCache;
switch (v.type()) {
case nThunk:
valueToCache = tree_cache::thunk_t{};
break;
case nNull:
case nList:
case nFunction:
case nExternal:
valueToCache = tree_cache::unknown_t{};
break;
case nBool:
valueToCache = v.boolean;
break;
case nString:
valueToCache = tree_cache::string_t{
v.string.s,
v.getContext()
};
break;
case nPath:
valueToCache = tree_cache::string_t{
v.path,
{}
};
break;
case nAttrs:
valueToCache = tree_cache::attributeSet_t{};
break;
case nInt:
valueToCache = v.integer;
break;
case nFloat:
valueToCache = v.fpoint;
break;
};
return valueToCache;
}
std::optional<std::vector<Symbol>> ValueCache::listChildren(SymbolTable& symbols)
{
auto ret = std::vector<Symbol>();
if (rawCache) {
auto cachedValue = rawCache->getCachedValue();
if (std::get_if<tree_cache::attributeSet_t>(&cachedValue)) {
for (auto & fieldStr : rawCache->getChildren())
ret.push_back(symbols.create(fieldStr));
}
return ret;
}
return std::nullopt;
}
std::optional<std::vector<Symbol>> ValueCache::listChildrenAtPath(SymbolTable & symbols, const std::vector<Symbol> & attrPath)
{
auto ret = std::vector<Symbol>();
if (rawCache) {
auto rawChildren = rawCache->getChildrenAtPath(attrPath);
if (!rawChildren) return std::nullopt;
for (auto & fieldStr : rawChildren.value())
ret.push_back(symbols.create(fieldStr));
return ret;
}
return std::nullopt;
}
std::vector<Symbol> EvalState::listAttrFields(Value & attrs, const Pos & pos)
{
// First try to get it from the cache
if (auto cachedRes = attrs.getCache().listChildren(symbols))
return cachedRes.value();
auto ret = std::vector<Symbol>();
forceAttrs(attrs, pos);
ret.reserve(attrs.attrs->size());
for (auto & attr : *attrs.attrs) {
ret.push_back(attr.name);
}
return ret;
}
EvalState::AttrAccesResult EvalState::getOptionalAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos)
{
Value* resValue = allocValue();
auto accessResult = getOptionalAttrField(attrs, selector, pos, *resValue);
accessResult.value = resValue;
return accessResult;
}
ValueCache::CacheResult ValueCache::getValue(Store & store, const std::vector<Symbol> & selector, Value & dest)
{
if (!rawCache)
return { NoCacheKey };
auto resultingCursor = rawCache->findAlongAttrPath(selector);
if (!resultingCursor)
return { CacheMiss };
dest.setCache(ValueCache(resultingCursor));
auto cachedValue = resultingCursor->getCachedValue();
return std::visit(
overloaded{
[&](tree_cache::attributeSet_t) { return ValueCache::CacheResult{ Forward }; },
[&](tree_cache::unknown_t) { return ValueCache::CacheResult{ UnCacheable }; },
[&](tree_cache::thunk_t) { return ValueCache::CacheResult{ CacheMiss }; },
[&](tree_cache::failed_t x) -> ValueCache::CacheResult {throw EvalError(x.error); },
[&](tree_cache::missing_t x) {
return ValueCache::CacheResult{
.returnCode = CacheHit,
.lastQueriedSymbolIfMissing = x.attrName
};
},
[&](tree_cache::string_t s) {
PathSet context;
for (auto& [pathName, outputName] : s.second) {
// If the cached value depends on some non-existent
// path, we need to discard it and force the evaluation
// to bring back the context in the store
if (!store.isValidPath(
store.parseStorePath(pathName)))
return ValueCache::CacheResult{UnCacheable};
context.insert("!" + outputName + "!" + pathName);
}
mkString(dest, s.first, context);
return ValueCache::CacheResult{CacheHit};
},
[&](bool b) {
dest.mkBool(b);
return ValueCache::CacheResult{CacheHit};
},
},
cachedValue);
}
struct ExprCastedVar : Expr
{
Value * v;
ExprCastedVar(Value * v) : v(v) {};
void show(std::ostream & str) const override {
std::set<const Value*> active;
printValue(str, active, *v);
}
void bindVars(const StaticEnv & env) override {}
void eval(EvalState & state, Env & env, Value & v) override {
v = std::move(*this->v);
}
Value * maybeThunk(EvalState & state, Env & env) override {
return v;
}
};
EvalState::AttrAccesResult EvalState::lazyGetOptionalAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos)
{
Value * dest = allocValue();
auto evalCache = attrs.getCache();
auto cacheResult = evalCache.getValue(*store, selector, *dest);
if (cacheResult.returnCode == ValueCache::CacheHit) {
if (cacheResult.lastQueriedSymbolIfMissing)
return {
.error =
AttrAccessError{*cacheResult.lastQueriedSymbolIfMissing},
.pos = &pos,
};
else
return {.pos = &pos, .value = dest};
}
auto & newEnv(allocEnv(0));
auto recordAsVar = new ExprCastedVar(&attrs);
auto accessExpr = new ExprSelect(pos, recordAsVar, selector);
dest->mkThunk (&newEnv, accessExpr);
return {
.pos = &pos,
.value = dest,
};
}
EvalState::AttrAccesResult EvalState::getOptionalAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest)
{
auto evalCache = attrs.getCache();
if (auto maybeRawVal = evalCache.getRawValue();
maybeRawVal.has_value() &&
!std::holds_alternative<tree_cache::attributeSet_t>(maybeRawVal.value()) &&
!std::holds_alternative<tree_cache::thunk_t>(maybeRawVal.value())
)
return {
.error = AttrAccessError{
.attrName = selector[0],
.illTypedValue = &attrs,
},
.pos = &pos,
};
auto cacheResult = evalCache.getValue(*store, selector, dest);
if (cacheResult.returnCode == ValueCache::CacheHit) {
if (cacheResult.lastQueriedSymbolIfMissing)
return {
.error =
AttrAccessError{*cacheResult.lastQueriedSymbolIfMissing},
.pos = &pos,
};
else
return {.pos = &pos, .value = &dest};
}
const Pos * pos2 = &pos;
Value * currentValue = &attrs;
forceValue(*currentValue, *pos2);
auto resultingCursor = evalCache;
for (auto & name : selector) {
nrLookups++;
Bindings::iterator j;
if (currentValue->type() != nAttrs)
return {
.error = AttrAccessError{
.attrName = name,
.illTypedValue = currentValue,
},
.pos = pos2,
};
if ((j = currentValue->attrs->find(name)) == currentValue->attrs->end())
return {.error = AttrAccessError { name }, .pos = pos2 };
currentValue = j->value;
pos2 = j->pos;
try {
forceValue(*currentValue, *pos2);
} catch (EvalError & e) {
resultingCursor.addFailedChild(name, e);
throw;
};
resultingCursor = resultingCursor.addChild(name, *currentValue);
if (cacheResult.returnCode == ValueCache::CacheMiss && currentValue->type() == nAttrs) {
resultingCursor.addAttrSetChilds(*currentValue->attrs);
}
currentValue->setCache(resultingCursor);
if (countCalls && pos2) attrSelects[*pos2]++;
}
dest = *currentValue;
return { .pos = pos2 };
}
const ValueCache ValueCache::empty = ValueCache(nullptr);
std::optional<tree_cache::AttrValue> ValueCache::getRawValue()
{
if (!rawCache)
return std::nullopt;
return rawCache->getCachedValue();
}
ValueCache ValueCache::addChild(const Symbol& name, const Value& value)
{
if (!rawCache)
return ValueCache::empty;
return ValueCache(rawCache->addChild(name, cachedValueFor(value)));
}
ValueCache ValueCache::addFailedChild(const Symbol& name, const Error & error)
{
if (!rawCache) return ValueCache::empty;
return ValueCache(rawCache->addChild(name, tree_cache::failed_t{ .error = error.msg() }));
}
ValueCache ValueCache::addNumChild(SymbolTable & symbols, int idx, const Value & value)
{
return addChild(symbols.create(std::to_string(idx)), value);
}
void ValueCache::addAttrSetChilds(Bindings & children)
{
if (!rawCache) return;
for (auto & attr : children) {
addChild(attr.name, *attr.value);
}
}
void ValueCache::addListChilds(SymbolTable & symbols, Value** elems, int listSize)
{
if (!rawCache) return;
for (auto i = 0; i < listSize; i++) {
addNumChild(symbols, i, *elems[i]);
}
}
void EvalState::getAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest)
{
auto missingFieldInfo = getOptionalAttrField(attrs, selector, pos, dest);
if (missingFieldInfo.error) {
if (missingFieldInfo.error->illTypedValue) {
throwTypeError("value is %1% while a set was expected", **missingFieldInfo.error->illTypedValue);
} else {
throwEvalError(
*missingFieldInfo.pos,
"attribute '%1%' missing", missingFieldInfo.error->attrName
);
}
}
}
Value* EvalState::getAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos)
{
Value* res = allocValue();
getAttrField(attrs, selector, pos, *res);
return res;
}
void ExprSelect::eval(EvalState & state, Env & env, Value & v)
{
Value vTmp;
Pos * pos2 = 0;
Value * vAttrs = &vTmp;
e->eval(state, env, vTmp);
try {
for (auto & i : attrPath) {
nrLookups++;
Bindings::iterator j;
Symbol name = getName(i, state, env);
if (def) {
state.forceValue(*vAttrs, pos);
if (vAttrs->type() != nAttrs ||
(j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
{
def->eval(state, env, v);
return;
}
} else {
state.forceAttrs(*vAttrs, pos);
if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
throwEvalError(pos, "attribute '%1%' missing", name);
}
vAttrs = j->value;
pos2 = j->pos;
if (state.countCalls && pos2) state.attrSelects[*pos2]++;
}
state.forceValue(*vAttrs, ( pos2 != NULL ? *pos2 : this->pos ) );
} catch (Error & e) {
if (pos2 && pos2->file != state.sDerivationNix)
addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'",
showAttrPath(state, env, attrPath));
throw;
std::vector<Symbol> evaluatedAttrs;
for (auto & i : attrPath) {
evaluatedAttrs.push_back(getName(i, state, env));
}
v = *vAttrs;
try {
if (def) {
auto missingFieldInfo = state.getOptionalAttrField(vTmp, evaluatedAttrs, pos, v);
if (missingFieldInfo.error) {
def->eval(state, env, v);
return;
}
return;
}
state.getAttrField(vTmp, evaluatedAttrs, pos, v);
} catch (Error & e) {
if (pos.file != state.sDerivationNix)
addErrorTrace(e, pos, "while evaluating the attribute '%1%'",
showAttrPath(state, env, attrPath));
throw;
}
}
@ -1690,18 +1997,6 @@ string EvalState::forceString(Value & v, const Pos & pos)
}
/* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */
std::pair<string, string> decodeContext(std::string_view s)
{
if (s.at(0) == '!') {
size_t index = s.find("!", 1);
return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
} else
return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
}
void copyContext(const Value & v, PathSet & context)
{
if (v.string.context)
@ -1710,7 +2005,7 @@ void copyContext(const Value & v, PathSet & context)
}
std::vector<std::pair<Path, std::string>> Value::getContext()
std::vector<std::pair<Path, std::string>> Value::getContext() const
{
std::vector<std::pair<Path, std::string>> res;
assert(internalType == tString);
@ -1720,6 +2015,15 @@ std::vector<std::pair<Path, std::string>> Value::getContext()
return res;
}
void Value::setCache(ValueCache cache)
{
evalCache = cache;
}
ValueCache Value::getCache() const
{
return evalCache;
}
string EvalState::forceString(Value & v, PathSet & context, const Pos & pos)
{
@ -1746,12 +2050,12 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos)
bool EvalState::isDerivation(Value & v)
{
if (v.type() != nAttrs) return false;
Bindings::iterator i = v.attrs->find(sType);
if (i == v.attrs->end()) return false;
forceValue(*i->value);
if (i->value->type() != nString) return false;
return strcmp(i->value->string.s, "derivation") == 0;
auto typeAccesRes = getOptionalAttrField(v, {sType}, noPos);
if (typeAccesRes.error)
return false;
forceValue(*typeAccesRes.value);
if (typeAccesRes.value->type() != nString) return false;
return strcmp(typeAccesRes.value->string.s, "derivation") == 0;
}
@ -1790,9 +2094,9 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
if (maybeString) {
return *maybeString;
}
auto i = v.attrs->find(sOutPath);
if (i == v.attrs->end()) throwTypeError(pos, "cannot coerce a set to a string");
return coerceToString(pos, *i->value, context, coerceMore, copyToStore);
auto accessResult = getOptionalAttrField(v, {sOutPath}, pos);
if (accessResult.error) throwTypeError(pos, "cannot coerce a set to a string");
return coerceToString(pos, *accessResult.value, context, coerceMore, copyToStore);
}
if (v.type() == nExternal)
@ -2103,5 +2407,19 @@ EvalSettings evalSettings;
static GlobalConfig::Register rEvalSettings(&evalSettings);
std::shared_ptr<tree_cache::Cache> EvalState::openTreeCache(Hash cacheKey)
{
if (auto iter = evalCache.find(cacheKey); iter != evalCache.end())
return iter->second;
if (!(evalSettings.useEvalCache && evalSettings.pureEval))
return nullptr;
auto thisCache = tree_cache::Cache::tryCreate(
cacheKey,
symbols
);
evalCache.insert({cacheKey, thisCache});
return thisCache;
}
}

32
src/libexpr/eval.hh
View File

@ -1,10 +1,14 @@
#pragma once
#include "attr-set.hh"
#include "context.hh"
#include "value.hh"
#include "nixexpr.hh"
#include "symbol-table.hh"
#include "config.hh"
#include "hash.hh"
#include "sqlite.hh"
#include "tree-cache.hh"
#include <map>
#include <optional>
@ -14,6 +18,9 @@
namespace nix {
namespace eval_cache {
class EvalCache;
}
class Store;
class EvalState;
@ -116,6 +123,8 @@ private:
#endif
FileEvalCache fileEvalCache;
std::map<Hash, std::shared_ptr<tree_cache::Cache>> evalCache;
SearchPath searchPath;
std::map<std::string, std::pair<bool, std::string>> searchPathResolved;
@ -131,6 +140,9 @@ public:
EvalState(const Strings & _searchPath, ref<Store> store);
~EvalState();
std::shared_ptr<eval_cache::EvalCache> openCache(Hash, std::function<Value *()> rootLoader);
std::shared_ptr<tree_cache::Cache> openTreeCache(Hash);
void addToSearchPath(const string & s);
SearchPath getSearchPath() { return searchPath; }
@ -303,6 +315,26 @@ public:
void mkThunk_(Value & v, Expr * expr);
void mkPos(Value & v, Pos * pos);
struct AttrAccessError {
const Symbol attrName;
// To distinguish between a missing field in an attribute set
// and an access to something that's not an attribute set
std::optional<Value* > illTypedValue;
};
struct AttrAccesResult {
std::optional<AttrAccessError> error;
const Pos * pos;
Value* value;
};
AttrAccesResult getOptionalAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest);
AttrAccesResult getOptionalAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos);
AttrAccesResult lazyGetOptionalAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos);
void getAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest);
Value* getAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos);
std::vector<Symbol> listAttrFields(Value & attrs, const Pos & pos);
void concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos);
/* Print statistics. */

25
src/libexpr/flake/flake.cc
View File

@ -617,9 +617,13 @@ void callFlake(EvalState & state,
, "/"), **vCallFlake);
}
state.callFunction(**vCallFlake, *vLocks, *vTmp1, noPos);
state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
vTmp1->mkApp(*vCallFlake, vLocks);
vTmp2->mkApp(vTmp1, vRootSrc);
vRes.mkApp(vTmp2, vRootSubdir);
auto fingerprint = lockedFlake.getFingerprint();
auto evalCache = state.openTreeCache(fingerprint);
auto cacheRoot = evalCache ? evalCache->getRoot() : nullptr;
vRes.setCache(ValueCache(cacheRoot));
}
static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
@ -658,4 +662,19 @@ Fingerprint LockedFlake::getFingerprint() const
Flake::~Flake() { }
Value* getFlakeValue(EvalState & state, const flake::LockedFlake lockedFlake)
{
/* For testing whether the evaluation cache is
complete. */
if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0")
throw Error("not everything is cached, but evaluation is not allowed");
auto vFlake = state.allocValue();
flake::callFlake(state, lockedFlake, *vFlake);
auto aOutputs = state.getAttrField(*vFlake, {state.symbols.create("outputs")}, noPos);
return aOutputs;
}
}

2
src/libexpr/flake/flake.hh
View File

@ -6,7 +6,6 @@
#include "value.hh"
namespace nix {
class EvalState;
namespace fetchers { struct Tree; }
@ -139,4 +138,5 @@ void emitTreeAttrs(
const fetchers::Input & input,
Value & v, bool emptyRevFallback = false);
Value* getFlakeValue(EvalState & state, const flake::LockedFlake lockedFlake);
}

77
src/libexpr/get-drvs.cc
View File

@ -10,14 +10,14 @@
namespace nix {
DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs)
DrvInfo::DrvInfo(EvalState & state, const string & attrPath, std::optional<Value> attrs)
: state(&state), attrs(attrs), attrPath(attrPath)
{
}
DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs)
: state(&state), attrs(nullptr), attrPath("")
: state(&state), attrs(std::nullopt), attrPath("")
{
auto [drvPath, selectedOutputs] = store->parsePathWithOutputs(drvPathWithOutputs);
@ -46,12 +46,28 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
}
std::optional<Value> DrvInfo::queryOptionalAttr(const Symbol attrName) const
{
// Erk
Value * mutableAttrs = const_cast<Value*>(&attrs.value());
auto accessResult = state->getOptionalAttrField(*mutableAttrs, {attrName}, noPos);
return accessResult.error ? std::nullopt : std::optional{*accessResult.value};
}
Value DrvInfo::queryAttr(const Symbol attrName) const
{
Value res;
// Erk
Value * mutableAttrs = const_cast<Value*>(&attrs.value());
state->getAttrField(*mutableAttrs, {attrName}, noPos, res);
return res;
}
string DrvInfo::queryName() const
{
if (name == "" && attrs) {
auto i = attrs->find(state->sName);
if (i == attrs->end()) throw TypeError("derivation name missing");
name = state->forceStringNoCtx(*i->value);
Value nameVal = queryAttr(state->sName);
name = state->forceStringNoCtx(nameVal);
}
return name;
}
@ -60,8 +76,8 @@ string DrvInfo::queryName() const
string DrvInfo::querySystem() const
{
if (system == "" && attrs) {
auto i = attrs->find(state->sSystem);
system = i == attrs->end() ? "unknown" : state->forceStringNoCtx(*i->value, *i->pos);
auto maybeSystemVal = queryOptionalAttr(state->sSystem);
system = maybeSystemVal ? state->forceStringNoCtx(*maybeSystemVal) : "unknown";
}
return system;
}
@ -70,9 +86,10 @@ string DrvInfo::querySystem() const
string DrvInfo::queryDrvPath() const
{
if (drvPath == "" && attrs) {
Bindings::iterator i = attrs->find(state->sDrvPath);
auto drvPathVal = queryOptionalAttr(state->sDrvPath);
PathSet context;
drvPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : "";
if (drvPathVal)
drvPath = state->coerceToPath(noPos, *drvPathVal, context);
}
return drvPath;
}
@ -81,10 +98,10 @@ string DrvInfo::queryDrvPath() const
string DrvInfo::queryOutPath() const
{
if (!outPath && attrs) {
Bindings::iterator i = attrs->find(state->sOutPath);
auto val = queryOptionalAttr(state->sOutPath);
PathSet context;
if (i != attrs->end())
outPath = state->coerceToPath(*i->pos, *i->value, context);
if (val)
outPath = state->coerceToPath(noPos, *val, context);
}
if (!outPath)
throw UnimplementedError("CA derivations are not yet supported");
@ -96,23 +113,23 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
{
if (outputs.empty()) {
/* Get the ‘outputs’ list. */
Bindings::iterator i;
if (attrs && (i = attrs->find(state->sOutputs)) != attrs->end()) {
state->forceList(*i->value, *i->pos);
std::optional<Value> outputsList;
if (attrs && (outputsList = queryOptionalAttr(state->sOutputs))) {
state->forceList(*outputsList);
/* For each output... */
for (unsigned int j = 0; j < i->value->listSize(); ++j) {
for (unsigned int j = 0; j < outputsList->listSize(); ++j) {
/* Evaluate the corresponding set. */
string name = state->forceStringNoCtx(*i->value->listElems()[j], *i->pos);
Bindings::iterator out = attrs->find(state->symbols.create(name));
if (out == attrs->end()) continue; // FIXME: throw error?
state->forceAttrs(*out->value);
string name = state->forceStringNoCtx(*outputsList->listElems()[j]);
auto out = queryOptionalAttr(state->symbols.create(name));
if (!out) continue;
state->forceAttrs(*out);
/* And evaluate its ‘outPath’ attribute. */
Bindings::iterator outPath = out->value->attrs->find(state->sOutPath);
if (outPath == out->value->attrs->end()) continue; // FIXME: throw error?
Value outPath;
state->getAttrField(*out, {state->sOutPath}, noPos, outPath);
PathSet context;
outputs[name] = state->coerceToPath(*outPath->pos, *outPath->value, context);
outputs[name] = state->coerceToPath(noPos, outPath, context);
}
} else
outputs["out"] = queryOutPath();
@ -140,8 +157,8 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
string DrvInfo::queryOutputName() const
{
if (outputName == "" && attrs) {
Bindings::iterator i = attrs->find(state->sOutputName);
outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value) : "";
auto val = queryOptionalAttr(state->sOutputName);
outputName = val ? state->forceStringNoCtx(*val) : "";
}
return outputName;
}
@ -151,10 +168,10 @@ Bindings * DrvInfo::getMeta()
{
if (meta) return meta;
if (!attrs) return 0;
Bindings::iterator a = attrs->find(state->sMeta);
if (a == attrs->end()) return 0;
state->forceAttrs(*a->value, *a->pos);
meta = a->value->attrs;
auto val = queryOptionalAttr(state->sMeta);
if (!val) return 0;
state->forceAttrs(*val);
meta = val->attrs;
return meta;
}
@ -285,7 +302,7 @@ static bool getDerivation(EvalState & state, Value & v,
derivation {...}; y = x;}'. */
if (!done.insert(v.attrs).second) return false;
DrvInfo drv(state, attrPath, v.attrs);
DrvInfo drv(state, attrPath, v);
drv.queryName();

8
src/libexpr/get-drvs.hh
View File

@ -26,7 +26,8 @@ private:
bool failed = false; // set if we get an AssertionError
Bindings * attrs = nullptr, * meta = nullptr;
std::optional<Value> attrs;
Bindings * meta = nullptr;
Bindings * getMeta();
@ -36,9 +37,12 @@ public:
string attrPath; /* path towards the derivation */
DrvInfo(EvalState & state) : state(&state) { };
DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs);
DrvInfo(EvalState & state, const string & attrPath, std::optional<Value> attrs);
DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs);
Value queryAttr(const Symbol attrName) const;
std::optional<Value> queryOptionalAttr(const Symbol attrName) const;
string queryName() const;
string querySystem() const;
string queryDrvPath() const;

4
src/libexpr/nixexpr.hh
View File

@ -166,6 +166,10 @@ struct ExprSelect : Expr
AttrPath attrPath;
ExprSelect(const Pos & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { };
ExprSelect(const Pos & pos, Expr * e, const Symbol & name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); };
ExprSelect(const Pos & pos, Expr * e, const std::vector<Symbol> & names) : pos(pos), e(e), def(0) {
for (auto & name : names)
attrPath.push_back(AttrName(name));
};
COMMON_METHODS
};

138
src/libexpr/primops.cc
View File

@ -410,7 +410,7 @@ static RegisterPrimOp primop_isNull({
Return `true` if *e* evaluates to `null`, and `false` otherwise.
> **Warning**
>
>
> This function is *deprecated*; just write `e == null` instead.
)",
.fun = prim_isNull,
@ -813,17 +813,19 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
{
state.forceAttrs(*args[0], pos);
Value & arg = *(args[0]);
auto getOptionalAttr= [&](Symbol attrName) -> std::optional<Value> {
auto accessResult = state.getOptionalAttrField(arg, {attrName}, pos);
return accessResult.error ? std::nullopt : std::optional{*accessResult.value};
};
/* Figure out the name first (for stack backtraces). */
Bindings::iterator attr = args[0]->attrs->find(state.sName);
if (attr == args[0]->attrs->end())
throw EvalError({
.msg = hintfmt("required attribute 'name' missing"),
.errPos = pos
});
state.getAttrField(arg, {state.sName}, pos, v);
string drvName;
Pos & posDrvName(*attr->pos);
auto posDrvName = pos; // FIXME: Should be tho position of the `name` attribute
try {
drvName = state.forceStringNoCtx(*attr->value, pos);
drvName = state.forceStringNoCtx(v, pos);
} catch (Error & e) {
e.addTrace(posDrvName, "while evaluating the derivation attribute 'name'");
throw;
@ -832,15 +834,15 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
/* Check whether attributes should be passed as a JSON file. */
std::ostringstream jsonBuf;
std::unique_ptr<JSONObject> jsonObject;
attr = args[0]->attrs->find(state.sStructuredAttrs);
if (attr != args[0]->attrs->end() && state.forceBool(*attr->value, pos))
if (auto maybeRawObj = getOptionalAttr(state.sStructuredAttrs)) {
if (state.forceBool(*maybeRawObj, pos))
jsonObject = std::make_unique<JSONObject>(jsonBuf);
}
/* Check whether null attributes should be ignored. */
bool ignoreNulls = false;
attr = args[0]->attrs->find(state.sIgnoreNulls);
if (attr != args[0]->attrs->end())
ignoreNulls = state.forceBool(*attr->value, pos);
if (auto maybeRawIgnoreNull = getOptionalAttr(state.sIgnoreNulls))
ignoreNulls = state.forceBool(*maybeRawIgnoreNull, pos);
/* Build the derivation expression by processing the attributes. */
Derivation drv;
@ -856,9 +858,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
StringSet outputs;
outputs.insert("out");
for (auto & i : args[0]->attrs->lexicographicOrder()) {
if (i->name == state.sIgnoreNulls) continue;
const string & key = i->name;
for (auto & currentArg : args[0]->attrs->lexicographicOrder()) {
if (currentArg->name == state.sIgnoreNulls) continue;
const string & key = currentArg->name;
vomit("processing attribute '%1%'", key);
auto handleHashMode = [&](const std::string & s) {
@ -900,22 +902,26 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
try {
Value argValue;
state.getAttrField(arg, {currentArg->name}, pos, argValue);
if (ignoreNulls) {
state.forceValue(*i->value, pos);
if (i->value->type() == nNull) continue;
state.forceValue(argValue, pos);
if (argValue.type() == nNull) continue;
}
if (i->name == state.sContentAddressed) {
if (currentArg->name == state.sContentAddressed) {
settings.requireExperimentalFeature("ca-derivations");
contentAddressed = state.forceBool(*i->value, pos);
contentAddressed = state.forceBool(argValue, pos);
}
/* The `args' attribute is special: it supplies the
command-line arguments to the builder. */
else if (i->name == state.sArgs) {
state.forceList(*i->value, pos);
for (unsigned int n = 0; n < i->value->listSize(); ++n) {
string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true);
else if (currentArg->name == state.sArgs) {
state.forceList(argValue, pos);
for (unsigned int n = 0; n < argValue.listSize(); ++n) {
string s = state.coerceToString(posDrvName, *argValue.listElems()[n], context, true);
drv.args.push_back(s);
}
}
@ -926,39 +932,39 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (jsonObject) {
if (i->name == state.sStructuredAttrs) continue;
if (currentArg->name == state.sStructuredAttrs) continue;
auto placeholder(jsonObject->placeholder(key));
printValueAsJSON(state, true, *i->value, placeholder, context);
printValueAsJSON(state, true, argValue, placeholder, context);
if (i->name == state.sBuilder)
drv.builder = state.forceString(*i->value, context, posDrvName);
else if (i->name == state.sSystem)
drv.platform = state.forceStringNoCtx(*i->value, posDrvName);
else if (i->name == state.sOutputHash)
outputHash = state.forceStringNoCtx(*i->value, posDrvName);
else if (i->name == state.sOutputHashAlgo)
outputHashAlgo = state.forceStringNoCtx(*i->value, posDrvName);
else if (i->name == state.sOutputHashMode)
handleHashMode(state.forceStringNoCtx(*i->value, posDrvName));
else if (i->name == state.sOutputs) {
if (currentArg->name == state.sBuilder)
drv.builder = state.forceString(argValue, context, posDrvName);
else if (currentArg->name == state.sSystem)
drv.platform = state.forceStringNoCtx(argValue, posDrvName);
else if (currentArg->name == state.sOutputHash)
outputHash = state.forceStringNoCtx(argValue, posDrvName);
else if (currentArg->name == state.sOutputHashAlgo)
outputHashAlgo = state.forceStringNoCtx(argValue, posDrvName);
else if (currentArg->name == state.sOutputHashMode)
handleHashMode(state.forceStringNoCtx(argValue, posDrvName));
else if (currentArg->name == state.sOutputs) {
/* Require ‘outputs’ to be a list of strings. */
state.forceList(*i->value, posDrvName);
state.forceList(argValue, posDrvName);
Strings ss;
for (unsigned int n = 0; n < i->value->listSize(); ++n)
ss.emplace_back(state.forceStringNoCtx(*i->value->listElems()[n], posDrvName));
for (unsigned int n = 0; n < argValue.listSize(); ++n)
ss.emplace_back(state.forceStringNoCtx(*argValue.listElems()[n], posDrvName));
handleOutputs(ss);
}
} else {
auto s = state.coerceToString(posDrvName, *i->value, context, true);
auto s = state.coerceToString(posDrvName, argValue, context, true);
drv.env.emplace(key, s);
if (i->name == state.sBuilder) drv.builder = s;
else if (i->name == state.sSystem) drv.platform = s;
else if (i->name == state.sOutputHash) outputHash = s;
else if (i->name == state.sOutputHashAlgo) outputHashAlgo = s;
else if (i->name == state.sOutputHashMode) handleHashMode(s);
else if (i->name == state.sOutputs)
if (currentArg->name == state.sBuilder) drv.builder = s;
else if (currentArg->name == state.sSystem) drv.platform = s;
else if (currentArg->name == state.sOutputHash) outputHash = s;
else if (currentArg->name == state.sOutputHashAlgo) outputHashAlgo = s;
else if (currentArg->name == state.sOutputHashMode) handleHashMode(s);
else if (currentArg->name == state.sOutputs)
handleOutputs(tokenizeString<Strings>(s));
}
@ -1918,26 +1924,26 @@ static RegisterPrimOp primop_path({
An enrichment of the built-in path type, based on the attributes
present in *args*. All are optional except `path`:
- path
- path
The underlying path.
- name
- name
The name of the path when added to the store. This can used to
reference paths that have nix-illegal characters in their names,
like `@`.
- filter
- filter
A function of the type expected by `builtins.filterSource`,
with the same semantics.
- recursive
- recursive
When `false`, when `path` is added to the store it is with a
flat hash, rather than a hash of the NAR serialization of the
file. Thus, `path` must refer to a regular file, not a
directory. This allows similar behavior to `fetchurl`. Defaults
to `true`.
- sha256
- sha256
When provided, this is the expected hash of the file at the
path. Evaluation will fail if the hash is incorrect, and
providing a hash allows `builtins.path` to be used even when the
@ -1956,13 +1962,14 @@ static RegisterPrimOp primop_path({
strings. */
static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
auto attrNames = state.listAttrFields(*args[0], pos);
state.forceAttrs(*args[0], pos);
state.mkList(v, args[0]->attrs->size());
state.mkList(v, attrNames.size());
size_t n = 0;
for (auto & i : *args[0]->attrs)
mkString(*(v.listElems()[n++] = state.allocValue()), i.name);
for (auto & name : attrNames)
mkString(*(v.listElems()[n++] = state.allocValue()), name);
std::sort(v.listElems(), v.listElems() + n,
[](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; });
@ -2013,17 +2020,8 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
string attr = state.forceStringNoCtx(*args[0], pos);
state.forceAttrs(*args[1], pos);
state.getAttrField(*args[1], {state.symbols.create(attr)}, pos, v);
// !!! Should we create a symbol here or just do a lookup?
Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr));
if (i == args[1]->attrs->end())
throw EvalError({
.msg = hintfmt("attribute '%1%' missing", attr),
.errPos = pos
});
// !!! add to stack trace?
if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
state.forceValue(*i->value, pos);
v = *i->value;
}
static RegisterPrimOp primop_getAttr({
@ -2356,7 +2354,13 @@ static RegisterPrimOp primop_isList({
static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Value & v)
{
auto evalCache = list.getCache();
auto atSymbol = state.symbols.create(std::to_string(n));
auto cacheResult = evalCache.getValue(*state.store, {atSymbol}, v);
if (cacheResult.returnCode == ValueCache::CacheHit) return;
state.forceList(list, pos);
evalCache.addListChilds(state.symbols, list.listElems(), list.listSize());
if (n < 0 || (unsigned int) n >= list.listSize())
throw Error({
.msg = hintfmt("list index %1% is out of bounds", n),
@ -2424,7 +2428,7 @@ static RegisterPrimOp primop_tail({
the argument isn’t a list or is an empty list.
> **Warning**
>
>
> This function should generally be avoided since it's inefficient:
> unlike Haskell's `tail`, it takes O(n) time, so recursing over a
> list by repeatedly calling `tail` takes O(n^2) time.

397
src/libexpr/tree-cache.cc Normal file
View File

@ -0,0 +1,397 @@
#include "tree-cache.hh"
#include "sqlite.hh"
#include "store-api.hh"
#include "context.hh"
namespace nix::tree_cache {
static const char * schema = R"sql(
create table if not exists Attributes (
id integer primary key autoincrement not null,
parent integer not null,
name text,
type integer not null,
value text,
context text,
unique (parent, name)
);
create index if not exists IndexByParent on Attributes(parent, name);
)sql";
struct AttrDb
{
std::atomic_bool failed{false};
struct State
{
SQLite db;
SQLiteStmt insertAttribute;
SQLiteStmt updateAttribute;
SQLiteStmt insertAttributeWithContext;
SQLiteStmt queryAttribute;
SQLiteStmt queryAttributes;
std::unique_ptr<SQLiteTxn> txn;
};
std::unique_ptr<Sync<State>> _state;
AttrDb(const Hash & fingerprint)
: _state(std::make_unique<Sync<State>>())
{
auto state(_state->lock());
Path cacheDir = getCacheDir() + "/nix/eval-cache-v3";
createDirs(cacheDir);
Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
state->db = SQLite(dbPath);
state->db.isCache();
state->db.exec(schema);
state->insertAttribute.create(state->db,
"insert into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
state->updateAttribute.create(state->db,
"update Attributes set type = ?, value = ?, context = ? where id = ?");
state->insertAttributeWithContext.create(state->db,
"insert into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)");
state->queryAttribute.create(state->db,
"select id, type, value, context from Attributes where parent = ? and name = ?");
state->queryAttributes.create(state->db,
"select name from Attributes where parent = ?");
state->txn = std::make_unique<SQLiteTxn>(state->db);
}
~AttrDb()
{
try {
auto state(_state->lock());
if (!failed)
state->txn->commit();
state->txn.reset();
} catch (...) {
ignoreException();
}
}
template<typename F>
AttrId doSQLite(F && fun)
{
if (failed) return 0;
try {
return fun();
} catch (SQLiteError &) {
ignoreException();
failed = true;
return 0;
}
}
/**
* Store a leaf of the tree in the db
*/
AttrId addEntry(
const AttrKey & key,
const AttrValue & value)
{
return doSQLite([&]()
{
auto state(_state->lock());
auto rawValue = RawValue::fromVariant(value);
state->insertAttributeWithContext.use()
(key.first)
(key.second)
(rawValue.type)
(rawValue.value.value_or(""), rawValue.value.has_value())
(rawValue.serializeContext())
.exec();
AttrId rowId = state->db.getLastInsertedRowId();
assert(rowId);
return rowId;
});
}
AttrId setBool(
AttrKey key,
bool b)
{
return addEntry(key, b);
}
std::optional<AttrId> getId(const AttrKey& key)
{
auto state(_state->lock());
auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
if (!queryAttribute.next()) return std::nullopt;
return (AttrType) queryAttribute.getInt(0);
}
AttrId setOrUpdate(const AttrKey& key, const AttrValue& value)
{
debug("cache: miss for the attribute %s", key.second);
if (auto existingId = getId(key)) {
setValue(*existingId, value);
return *existingId;
}
return addEntry(key, value);
}
void setValue(const AttrId & id, const AttrValue & value)
{
auto state(_state->lock());
auto rawValue = RawValue::fromVariant(value);
state->updateAttribute.use()
(rawValue.type)
(rawValue.value.value_or(""), rawValue.value.has_value())
(rawValue.serializeContext())
(id)
.exec();
}
std::optional<std::pair<AttrId, AttrValue>> getValue(AttrKey key)
{
auto state(_state->lock());
auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
if (!queryAttribute.next()) return {};
auto rowId = (AttrType) queryAttribute.getInt(0);
auto type = (AttrType) queryAttribute.getInt(1);
switch (type) {
case AttrType::Attrs: {
return {{rowId, attributeSet_t()}};
}
case AttrType::String: {
std::vector<std::pair<Path, std::string>> context;
if (!queryAttribute.isNull(3))
for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
context.push_back(decodeContext(s));
return {{rowId, string_t{queryAttribute.getStr(2), context}}};
}
case AttrType::Bool:
return {{rowId, queryAttribute.getInt(2) != 0}};
case AttrType::Int:
return {{rowId, queryAttribute.getInt(2)}};
case AttrType::Double:
return {{rowId, queryAttribute.getInt(2)}};
case AttrType::Unknown:
return {{rowId, unknown_t{}}};
case AttrType::Thunk:
return {{rowId, thunk_t{}}};
case AttrType::Missing:
return {{rowId, missing_t{key.second}}};
case AttrType::Failed:
return {{rowId, failed_t{queryAttribute.getStr(2)}}};
default:
throw Error("unexpected type in evaluation cache");
}
}
std::vector<std::string> getChildren(AttrId parentId)
{
std::vector<std::string> res;
auto state(_state->lock());
auto queryAttributes(state->queryAttributes.use()(parentId));
while (queryAttributes.next())
res.push_back(queryAttributes.getStr(0));
return res;
}
};
Cache::Cache(const Hash & useCache,
SymbolTable & symbols)
: db(std::make_shared<AttrDb>(useCache))
, symbols(symbols)
, rootSymbol(symbols.create(""))
{
}
std::shared_ptr<Cache> Cache::tryCreate(const Hash & useCache, SymbolTable & symbols)
{
try {
return std::make_shared<Cache>(useCache, symbols);
} catch (SQLiteError &) {
ignoreException();
return nullptr;
}
}
void Cache::commit()
{
if (db) {
debug("Saving the cache");
auto state(db->_state->lock());
if (state->txn->active) {
state->txn->commit();
state->txn.reset();
state->txn = std::make_unique<SQLiteTxn>(state->db);
}
}
}
Cursor::Ref Cache::getRoot()
{
return new Cursor(ref(shared_from_this()), std::nullopt, thunk_t{});
}
Cursor::Cursor(
ref<Cache> root,
const Parent & parent,
const AttrValue& value
)
: root(root)
, parentId(parent ? std::optional{parent->first.cachedValue.first} : std::nullopt)
, label(parent ? parent->second : root->rootSymbol)
, cachedValue({root->db->setOrUpdate(getKey(), value), value})
{
}
Cursor::Cursor(
ref<Cache> root,
const Parent & parent,
const AttrId & id,
const AttrValue & value
)
: root(root)
, parentId(parent ? std::optional{parent->first.cachedValue.first} : std::nullopt)
, label(parent ? parent->second : root->rootSymbol)
, cachedValue({id, value})
{
}
AttrKey Cursor::getKey()
{
if (!parentId)
return {0, root->rootSymbol};
return {*parentId, label};
}
AttrValue Cursor::getCachedValue()
{
return cachedValue.second;
}
void Cursor::setValue(const AttrValue & v)
{
root->db->setValue(cachedValue.first, v);
cachedValue.second = v;
}
Cursor::Ref Cursor::addChild(const Symbol & attrPath, const AttrValue & v)
{
Parent parent = {{*this, attrPath}};
auto childCursor = new Cursor(
root,
parent,
v
);
return childCursor;
}
std::vector<std::string> Cursor::getChildren()
{
return root->db->getChildren(cachedValue.first);
}
std::optional<std::vector<std::string>> Cursor::getChildrenAtPath(const std::vector<Symbol> & attrPath)
{
auto cursorAtPath = findAlongAttrPath(attrPath);
if (cursorAtPath)
return cursorAtPath->getChildren();
return std::nullopt;
}
Cursor::Ref Cursor::maybeGetAttr(const Symbol & name)
{
auto rawAttr = root->db->getValue({cachedValue.first, name});
if (rawAttr) {
Parent parent = {{*this, name}};
debug("cache: hit for the attribute %s", cachedValue.first);
return new Cursor (
root, parent, rawAttr->first,
rawAttr->second);
}
if (std::holds_alternative<attributeSet_t>(cachedValue.second)) {
// If the parent is an attribute set but we're not present in the db,
// then we're not a member of this attribute set. So mark as missing
return addChild(name, missing_t{name});
}
return nullptr;
}
Cursor::Ref Cursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
{
auto currentCursor = this;
for (auto & currentAccessor : attrPath) {
currentCursor = currentCursor->maybeGetAttr(currentAccessor);
if (!currentCursor)
break;
if (std::holds_alternative<missing_t>(currentCursor->cachedValue.second))
break;
if (std::holds_alternative<failed_t>(currentCursor->cachedValue.second))
break;
}
return currentCursor;
}
const RawValue RawValue::fromVariant(const AttrValue & value)
{
RawValue res;
std::visit(overloaded{
[&](attributeSet_t x) { res.type = AttrType::Attrs; },
[&](string_t x) {
res.type = AttrType::String;
res.value = x.first;
res.context = x.second;
},
[&](bool x) {
res.type = AttrType::Bool;
res.value = x ? "1" : "0";
},
[&](int64_t x) {
res.type = AttrType::Int;
res.value = std::to_string(x);
},
[&](double x) {
res.type = AttrType::Double;
res.value = std::to_string(x);
},
[&](unknown_t x) { res.type = AttrType::Unknown; },
[&](missing_t x) { res.type = AttrType::Missing; },
[&](thunk_t x) { res.type = AttrType::Thunk; },
[&](failed_t x) {
res.type = AttrType::Failed;
res.value = x.error;
}
}, value);
return res;
}
std::string RawValue::serializeContext() const
{
std::string res;
for (auto & elt : context) {
res.append(encodeContext(elt.second, elt.first));
res.push_back(' ');
}
if (!res.empty())
res.pop_back(); // Remove the trailing space
return res;
}
}

148
src/libexpr/tree-cache.hh Normal file
View File

@ -0,0 +1,148 @@
/**
* caching for a tree-like data structure (like Nix values)
*
* The cache is an sqlite db whose rows are the nodes of the tree, with a
* pointer to their parent (except for the root of course)
*/
#pragma once
#include "sync.hh"
#include "hash.hh"
#include "symbol-table.hh"
#include <functional>
#include <variant>
namespace nix::tree_cache {
struct AttrDb;
class Cursor;
class Cache : public std::enable_shared_from_this<Cache>
{
private:
friend class Cursor;
/**
* The database holding the cache
*/
std::shared_ptr<AttrDb> db;
SymbolTable & symbols;
/**
* Distinguished symbol indicating the root of the tree
*/
const Symbol rootSymbol;
public:
Cache(
const Hash & useCache,
SymbolTable & symbols
);
static std::shared_ptr<Cache> tryCreate(const Hash & useCache, SymbolTable & symbols);
Cursor * getRoot();
/**
* Flush the cache to disk
*/
void commit();
};
enum AttrType {
Unknown = 0,
Attrs = 1,
String = 2,
Bool = 3,
Int = 4,
Double = 5,
Thunk = 6,
Missing = 7, // Missing fields of attribute sets
Failed = 8,
};
struct attributeSet_t {};
struct unknown_t {};
struct thunk_t {};
struct failed_t { string error; };
struct missing_t { Symbol attrName; };
typedef uint64_t AttrId;
typedef std::pair<AttrId, Symbol> AttrKey;
typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t;
typedef std::variant<
attributeSet_t,
string_t,
unknown_t,
thunk_t,
missing_t,
failed_t,
bool,
int64_t,
double
> AttrValue;
struct RawValue {
AttrType type;
std::optional<std::string> value;
std::vector<std::pair<Path, std::string>> context;
std::string serializeContext() const;
static const RawValue fromVariant(const AttrValue&);
AttrValue toVariant() const;
};
/**
* View inside the cache.
*
* A `Cursor` represents a node in the cached tree (be it a leaf or not)
*/
class Cursor : public std::enable_shared_from_this<Cursor>
{
/**
* The overall cache of which this cursor is a view
*/
ref<Cache> root;
typedef std::optional<std::pair<Cursor&, Symbol>> Parent;
std::optional<AttrId> parentId;
Symbol label;
std::pair<AttrId, AttrValue> cachedValue;
/**
* Get the identifier for this node in the database
*/
AttrKey getKey();
public:
using Ref = Cursor*;
// Create a new cache entry
Cursor(ref<Cache> root, const Parent & parent, const AttrValue&);
// Build a cursor from an existing cache entry
Cursor(ref<Cache> root, const Parent & parent, const AttrId& id, const AttrValue&);
AttrValue getCachedValue();
void setValue(const AttrValue & v);
Ref addChild(const Symbol & attrPath, const AttrValue & v);
Ref findAlongAttrPath(const std::vector<Symbol> & attrPath);
Ref maybeGetAttr(const Symbol & attrPath);
std::vector<std::string> getChildren();
std::optional<std::vector<std::string>> getChildrenAtPath(const std::vector<Symbol> & attrPath);
};
}

64
src/libexpr/value.hh
View File

@ -1,6 +1,7 @@
#pragma once
#include "symbol-table.hh"
#include "tree-cache.hh"
#if HAVE_BOEHMGC
#include <gc/gc_allocator.h>
@ -56,8 +57,51 @@ struct Pos;
class EvalState;
class XMLWriter;
class JSONPlaceholder;
class Store;
struct Value;
class ValueCache {
tree_cache::Cursor::Ref rawCache;
public:
ValueCache(tree_cache::Cursor::Ref rawCache) : rawCache(rawCache) {}
const static ValueCache empty;
enum ReturnCode {
// The cache result was an attribute set, so we forward it later in the
// chain
Forward,
CacheMiss,
CacheHit,
UnCacheable,
NoCacheKey,
};
struct CacheResult {
ReturnCode returnCode;
// In case the query returns a `missing_t`, the symbol that's missing
std::optional<Symbol> lastQueriedSymbolIfMissing;
};
CacheResult getValue(Store & store, const std::vector<Symbol> & selector, Value & dest);
ValueCache addChild(const Symbol & attrName, const Value & value);
ValueCache addFailedChild(const Symbol & attrName, const Error & error);
ValueCache addNumChild(SymbolTable & symbols, int idx, const Value & value);
void addAttrSetChilds(Bindings & children);
void addListChilds(SymbolTable & symbols, Value** elems, int listSize);
std::optional<std::vector<Symbol>> listChildren(SymbolTable&);
std::optional<std::vector<Symbol>> listChildrenAtPath(SymbolTable&, const std::vector<Symbol> & attrPath);
std::optional<tree_cache::AttrValue> getRawValue();
ValueCache() : rawCache(nullptr) {}
};
typedef int64_t NixInt;
typedef double NixFloat;
@ -113,6 +157,12 @@ friend std::string showType(const Value & v);
friend void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v);
public:
/*
* An optional evaluation cache (for flakes in particular).
* If this is set, then trying to get a value from this attrset will first
* try to get it from the cache
*/
ValueCache evalCache;
// Functions needed to distinguish the type
// These should be removed eventually, by putting the functionality that's
@ -346,7 +396,19 @@ public:
non-trivial. */
bool isTrivial() const;
std::vector<std::pair<Path, std::string>> getContext();
std::vector<std::pair<Path, std::string>> getContext() const;
/*
* Set the associated cache view for this value.
* This cache will be used to speed-up some operations like accessing
* attribute sets elements.
*/
void setCache(ValueCache);
/*
* Get the cache associated with this value, if any.
*/
ValueCache getCache() const;
};

2
src/nix-env/nix-env.cc
View File

@ -374,7 +374,7 @@ static void queryInstSources(EvalState & state,
std::string name(path.name());
DrvInfo elem(state, "", nullptr);
DrvInfo elem(state, "", std::nullopt);
elem.setName(name);
if (path.isDerivation()) {

20
src/nix/app.cc
View File

@ -1,26 +1,28 @@
#include "installables.hh"
#include "store-api.hh"
#include "eval-inline.hh"
#include "eval-cache.hh"
#include "names.hh"
namespace nix {
App Installable::toApp(EvalState & state)
{
auto [cursor, attrPath] = getCursor(state);
auto [value, pos, attrPath] = toValue(state);
auto type = cursor->getAttr("type")->getString();
auto type = state.forceStringNoCtx(*state.getAttrField(*value, {state.sType}, pos));
if (type == "app") {
auto [program, context] = cursor->getAttr("program")->getStringWithContext();
StringSet context;
auto program = state.forceString(*state.getAttrField(*value, {state.symbols.create("program")}, pos));
if (!state.store->isInStore(program))
throw Error("app program '%s' is not in the Nix store", program);
std::vector<StorePathWithOutputs> context2;
for (auto & [path, name] : context)
for (auto & rawCtxItem : context) {
auto [path, name] = decodeContext(rawCtxItem);
context2.push_back({state.store->parseStorePath(path), {name}});
}
return App {
.context = std::move(context2),
@ -29,10 +31,10 @@ App Installable::toApp(EvalState & state)
}
else if (type == "derivation") {
auto drvPath = cursor->forceDerivation();
auto outPath = cursor->getAttr(state.sOutPath)->getString();
auto outputName = cursor->getAttr(state.sOutputName)->getString();
auto name = cursor->getAttr(state.sName)->getString();
auto drvPath = state.store->parseStorePath(state.forceString(*state.getAttrField(*value, {state.sDrvPath}, pos)));
auto outPath = state.forceString(*state.getAttrField(*value, {state.sOutPath}, pos));
auto outputName = state.forceStringNoCtx(*state.getAttrField(*value, {state.sOutputName}, pos));
auto name = state.forceStringNoCtx(*state.getAttrField(*value, {state.sName}, pos));
return App {
.context = { { drvPath, {outputName} } },
.program = outPath + "/bin/" + DrvName(name).name,

2
src/nix/bundle.cc
View File

@ -92,7 +92,7 @@ struct CmdBundle : InstallableCommand
arg->attrs->sort();
auto vRes = evalState->allocValue();
evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos);
evalState->callFunction(*bundler.toValue(*evalState).value, *arg, *vRes, noPos);
if (!evalState->isDerivation(*vRes))
throw Error("the bundler '%s' does not produce a derivation", bundler.what());

2
src/nix/edit.cc
View File

@ -28,7 +28,7 @@ struct CmdEdit : InstallableCommand
{
auto state = getEvalState();
auto [v, pos] = installable->toValue(*state);
auto [v, pos, _] = installable->toValue(*state);
try {
pos = findDerivationFilename(*state, *v, installable->what());

2
src/nix/eval.cc
View File

@ -60,7 +60,7 @@ struct CmdEval : MixJSON, InstallableCommand
auto state = getEvalState();
auto [v, pos] = installable->toValue(*state);
auto [v, pos, _] = installable->toValue(*state);
PathSet context;
if (apply) {

35
src/nix/flake.cc
View File

@ -11,7 +11,6 @@
#include "fetchers.hh"
#include "registry.hh"
#include "json.hh"
#include "eval-cache.hh"
#include <nlohmann/json.hpp>
#include <queue>
@ -600,9 +599,8 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
Strings{templateName == "" ? "defaultTemplate" : templateName},
Strings(attrsPathPrefixes), lockFlags);
auto [cursor, attrPath] = installable.getCursor(*evalState);
auto templateDir = cursor->getAttr("path")->getString();
auto flakeValue = installable.toValue(*evalState);
auto templateDir = evalState->forceStringNoCtx(*evalState->getAttrField(*flakeValue.value, {evalState->symbols.create("path")}, flakeValue.pos));
assert(store->isInStore(templateDir));
@ -829,9 +827,9 @@ struct CmdFlakeShow : FlakeCommand
auto state = getEvalState();
auto flake = std::make_shared<LockedFlake>(lockFlake());
std::function<void(eval_cache::AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)> visit;
std::function<void(Value & value, Pos & pos, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)> visit;
visit = [&](eval_cache::AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)
visit = [&](Value & value, Pos & pos, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)
{
Activity act(*logger, lvlInfo, actUnknown,
fmt("evaluating '%s'", concatStringsSep(".", attrPath)));
@ -839,13 +837,13 @@ struct CmdFlakeShow : FlakeCommand
auto recurse = [&]()
{
logger->cout("%s", headerPrefix);
auto attrs = visitor.getAttrs();
auto attrs = state->listAttrFields(value, pos);
for (const auto & [i, attr] : enumerate(attrs)) {
bool last = i + 1 == attrs.size();
auto visitor2 = visitor.getAttr(attr);
auto value2 = state->getAttrField(value, {attr}, pos);
auto attrPath2(attrPath);
attrPath2.push_back(attr);
visit(*visitor2, attrPath2,
visit(*value2, pos, attrPath2,
fmt(ANSI_GREEN "%s%s" ANSI_NORMAL ANSI_BOLD "%s" ANSI_NORMAL, nextPrefix, last ? treeLast : treeConn, attr),
nextPrefix + (last ? treeNull : treeLine));
}
@ -853,7 +851,7 @@ struct CmdFlakeShow : FlakeCommand
auto showDerivation = [&]()
{
auto name = visitor.getAttr(state->sName)->getString();
auto name = state->forceStringNoCtx(*state->getAttrField(value, {state->sName}, pos));
/*
std::string description;
@ -895,14 +893,14 @@ struct CmdFlakeShow : FlakeCommand
|| (attrPath.size() == 3 && (attrPath[0] == "checks" || attrPath[0] == "packages"))
)
{
if (visitor.isDerivation())
if (state->isDerivation(value))
showDerivation();
else
throw Error("expected a derivation");
}
else if (attrPath.size() > 0 && attrPath[0] == "hydraJobs") {
if (visitor.isDerivation())
if (state->isDerivation(value))
showDerivation();
else
recurse();
@ -914,7 +912,7 @@ struct CmdFlakeShow : FlakeCommand
else if (!showLegacy)
logger->cout("%s: " ANSI_YELLOW "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix);
else {
if (visitor.isDerivation())
if (state->isDerivation(value))
showDerivation();
else if (attrPath.size() <= 2)
// FIXME: handle recurseIntoAttrs
@ -926,8 +924,8 @@ struct CmdFlakeShow : FlakeCommand
(attrPath.size() == 2 && attrPath[0] == "defaultApp") ||
(attrPath.size() == 3 && attrPath[0] == "apps"))
{
auto aType = visitor.maybeGetAttr("type");
if (!aType || aType->getString() != "app")
auto aType = state->getOptionalAttrField(value, {state->sType}, pos);
if (aType.error || state->forceStringNoCtx(*aType.value) != "app")
throw EvalError("not an app definition");
logger->cout("%s: app", headerPrefix);
}
@ -936,7 +934,8 @@ struct CmdFlakeShow : FlakeCommand
(attrPath.size() == 1 && attrPath[0] == "defaultTemplate") ||
(attrPath.size() == 2 && attrPath[0] == "templates"))
{
auto description = visitor.getAttr("description")->getString();
/* auto description = visitor.getAttr("description")->getString(); */
auto description = state->forceStringNoCtx(*state->getAttrField(value, {state->sDescription}, pos));
logger->cout("%s: template: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description);
}
@ -954,9 +953,9 @@ struct CmdFlakeShow : FlakeCommand
}
};
auto cache = openEvalCache(*state, flake);
auto flakeValue = getFlakeValue(*state, *flake);
visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), "");
visit(*flakeValue, noPos, {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), "");
}
};

36
src/nix/search.cc
View File

@ -7,7 +7,6 @@
#include "common-args.hh"
#include "json.hh"
#include "shared.hh"
#include "eval-cache.hh"
#include "attr-path.hh"
#include <regex>
@ -81,31 +80,38 @@ struct CmdSearch : InstallableCommand, MixJSON
uint64_t results = 0;
std::function<void(eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath, bool initialRecurse)> visit;
std::function<void(Value & cursor, const Pos& pos, const std::vector<Symbol> & attrPath, bool initialRecurse)> visit;
visit = [&](eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath, bool initialRecurse)
visit = [&](Value & cursor, const Pos& pos, const std::vector<Symbol> & attrPath, bool initialRecurse)
{
Activity act(*logger, lvlInfo, actUnknown,
fmt("evaluating '%s'", concatStringsSep(".", attrPath)));
try {
auto recurse = [&]()
{
for (const auto & attr : cursor.getAttrs()) {
auto cursor2 = cursor.getAttr(attr);
for (const auto & attr : evalState->listAttrFields(cursor, pos)) {
Value* cursor2;
try {
auto cursor2Res = evalState->lazyGetOptionalAttrField(cursor, {attr}, pos);
if (cursor2Res.error)
continue; // Shouldn't happen, but who knows
cursor2 = cursor2Res.value;
} catch (EvalError&) {
continue;
}
auto attrPath2(attrPath);
attrPath2.push_back(attr);
visit(*cursor2, attrPath2, false);
visit(*cursor2, pos, attrPath2, false);
}
};
if (cursor.isDerivation()) {
if (evalState->isDerivation(cursor)) {
size_t found = 0;
DrvName name(cursor.getAttr("name")->getString());
DrvName name(evalState->forceStringNoCtx(*evalState->getAttrField(cursor, {evalState->sName}, pos)));
auto aMeta = cursor.maybeGetAttr("meta");
auto aDescription = aMeta ? aMeta->maybeGetAttr("description") : nullptr;
auto description = aDescription ? aDescription->getString() : "";
auto descriptionGetRes = evalState->getOptionalAttrField(cursor, {evalState->sMeta, evalState->sDescription}, pos);
auto description = !descriptionGetRes.error ? state->forceStringNoCtx(*descriptionGetRes.value) : "";
std::replace(description.begin(), description.end(), '\n', ' ');
auto attrPath2 = concatStringsSep(".", attrPath);
@ -154,8 +160,8 @@ struct CmdSearch : InstallableCommand, MixJSON
recurse();
else if (attrPath[0] == "legacyPackages" && attrPath.size() > 2) {
auto attr = cursor.maybeGetAttr(state->sRecurseForDerivations);
if (attr && attr->getBool())
auto recurseForDrvGetRes = evalState->getOptionalAttrField(cursor, {evalState->sRecurseForDerivations}, pos);
if (!recurseForDrvGetRes.error && evalState->forceBool(*recurseForDrvGetRes.value, pos))
recurse();
}
@ -165,8 +171,8 @@ struct CmdSearch : InstallableCommand, MixJSON
}
};
for (auto & [cursor, prefix] : installable->getCursors(*state))
visit(*cursor, parseAttrPath(*state, prefix), true);
for (auto & [cursor, pos, prefix] : installable->toValues(*state))
visit(*cursor, pos, parseAttrPath(*state, prefix), true);
if (!json && !results)
throw Error("no results for the given search term(s)!");

2
tests/plugins/local.mk
View File

@ -8,4 +8,4 @@ libplugintest_ALLOW_UNDEFINED := 1
libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1
libplugintest_CXXFLAGS := -I src/libutil -I src/libexpr
libplugintest_CXXFLAGS := -I src/libutil -I src/libexpr -I src/libstore