From 6496fc1246333fe215a9939c0596b21fcceb617d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 3 Feb 2025 11:28:43 -0500 Subject: [PATCH] Move `exportReferencesGraph` to `DerivationOptions` --- .../derivation-advanced-attrs.cc | 8 ++-- src/libstore/build/derivation-goal.cc | 3 +- src/libstore/derivation-options.cc | 39 ++++++++++++++++++- .../include/nix/store/derivation-options.hh | 25 +++++++++++- .../include/nix/store/parsed-derivations.hh | 3 +- .../include/nix/store/restricted-store.hh | 2 +- src/libstore/misc.cc | 2 +- src/libstore/parsed-derivations.cc | 18 ++++----- .../unix/build/local-derivation-goal.cc | 28 ++----------- src/nix-build/nix-build.cc | 4 +- 10 files changed, 84 insertions(+), 48 deletions(-) diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index b8cfa2498..4f381ab33 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -82,7 +82,7 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_defaults) auto drvPath = writeDerivation(*store, got, NoRepair, true); ParsedDerivation parsedDrv(got); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + DerivationOptions options = DerivationOptions::fromParsedDerivation(*store, parsedDrv); EXPECT_TRUE(!parsedDrv.hasStructuredAttrs()); @@ -117,7 +117,7 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes) auto drvPath = writeDerivation(*store, got, NoRepair, true); ParsedDerivation parsedDrv(got); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + DerivationOptions options = DerivationOptions::fromParsedDerivation(*store, parsedDrv); StringSet systemFeatures{"rainbow", "uid-range"}; @@ -158,7 +158,7 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttr auto drvPath = writeDerivation(*store, got, NoRepair, true); ParsedDerivation parsedDrv(got); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + DerivationOptions options = DerivationOptions::fromParsedDerivation(*store, parsedDrv); EXPECT_TRUE(parsedDrv.hasStructuredAttrs()); @@ -192,7 +192,7 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttr auto drvPath = writeDerivation(*store, got, NoRepair, true); ParsedDerivation parsedDrv(got); - DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + DerivationOptions options = DerivationOptions::fromParsedDerivation(*store, parsedDrv); StringSet systemFeatures{"rainbow", "uid-range"}; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 8e43ab2ae..729b52d63 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -182,7 +182,8 @@ Goal::Co DerivationGoal::haveDerivation() parsedDrv = std::make_unique(*drv); try { - drvOptions = std::make_unique(DerivationOptions::fromParsedDerivation(*parsedDrv)); + drvOptions = std::make_unique( + DerivationOptions::fromParsedDerivation(worker.store, *parsedDrv)); } catch (Error & e) { e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath)); throw; diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index 962222f6d..841b91716 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -3,9 +3,11 @@ #include "nix/store/parsed-derivations.hh" #include "nix/util/types.hh" #include "nix/util/util.hh" + #include #include #include +#include namespace nix { @@ -13,7 +15,8 @@ using OutputChecks = DerivationOptions::OutputChecks; using OutputChecksVariant = std::variant>; -DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation & parsed, bool shouldWarn) +DerivationOptions +DerivationOptions::fromParsedDerivation(const StoreDirConfig & store, const ParsedDerivation & parsed, bool shouldWarn) { DerivationOptions defaults = {}; @@ -126,6 +129,39 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation } return res; }(), + .exportReferencesGraph = + [&] { + std::map ret; + + if (auto structuredAttrs = parsed.structuredAttrs.get()) { + auto e = optionalValueAt(*structuredAttrs, "exportReferencesGraph"); + if (!e || !e->is_object()) + return ret; + for (auto & [key, storePathsJson] : getObject(*e)) { + StorePathSet storePaths; + for (auto & p : storePathsJson) + storePaths.insert(store.toStorePath(p.get()).first); + ret.insert_or_assign(key, std::move(storePaths)); + } + } else { + auto s = getOr(parsed.drv.env, "exportReferencesGraph", ""); + Strings ss = tokenizeString(s); + if (ss.size() % 2 != 0) + throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); + for (Strings::iterator i = ss.begin(); i != ss.end();) { + auto fileName = std::move(*i++); + static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); + if (!std::regex_match(fileName, regex)) + throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); + + auto & storePathS = *i++; + if (!store.isInStore(storePathS)) + throw Error("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); + ret.insert_or_assign(std::move(fileName), StorePathSet{store.toStorePath(storePathS).first}); + } + } + return ret; + }(), .additionalSandboxProfile = parsed.getStringAttr("__sandboxProfile").value_or(defaults.additionalSandboxProfile), .noChroot = parsed.getBoolAttr("__noChroot", defaults.noChroot), @@ -180,7 +216,6 @@ bool DerivationOptions::useUidRange(const BasicDerivation & drv) const { return getRequiredSystemFeatures(drv).count("uid-range"); } - } namespace nlohmann { diff --git a/src/libstore/include/nix/store/derivation-options.hh b/src/libstore/include/nix/store/derivation-options.hh index 8f549b737..04ddeb72f 100644 --- a/src/libstore/include/nix/store/derivation-options.hh +++ b/src/libstore/include/nix/store/derivation-options.hh @@ -8,6 +8,7 @@ #include "nix/util/types.hh" #include "nix/util/json-impls.hh" +#include "nix/store/store-dir-config.hh" namespace nix { @@ -95,6 +96,27 @@ struct DerivationOptions */ StringSet passAsFile; + /** + * The `exportReferencesGraph' feature allows the references graph + * to be passed to a builder + * + * ### Legacy case + * + * Given a `name` `pathSet` key-value pair, the references graph of + * `pathSet` will be stored in a text file `name' in the temporary + * build directory. The text files have the format used by + * `nix-store + * --register-validity'. However, the `deriver` fields are left + * empty. + * + * ### "Structured attributes" case + * + * The same information will be put put in the final structured + * attributes give to the builder. The set of paths in the original JSON + * is replaced with a list of `PathInfo` in JSON format. + */ + std::map exportReferencesGraph; + /** * env: __sandboxProfile * @@ -152,7 +174,8 @@ struct DerivationOptions * (e.g. JSON) but is necessary for supporing old formats (e.g. * ATerm). */ - static DerivationOptions fromParsedDerivation(const ParsedDerivation & parsed, bool shouldWarn = true); + static DerivationOptions + fromParsedDerivation(const StoreDirConfig & store, const ParsedDerivation & parsed, bool shouldWarn = true); /** * @param drv Must be the same derivation we parsed this from. In diff --git a/src/libstore/include/nix/store/parsed-derivations.hh b/src/libstore/include/nix/store/parsed-derivations.hh index 2b1ab97d5..e23eab1bb 100644 --- a/src/libstore/include/nix/store/parsed-derivations.hh +++ b/src/libstore/include/nix/store/parsed-derivations.hh @@ -42,7 +42,8 @@ public: return static_cast(structuredAttrs); } - std::optional prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths); + std::optional + prepareStructuredAttrs(Store & store, const DerivationOptions & drvOptions, const StorePathSet & inputPaths); }; std::string writeStructuredAttrsShell(const nlohmann::json & json); diff --git a/src/libstore/include/nix/store/restricted-store.hh b/src/libstore/include/nix/store/restricted-store.hh index 84b455456..67c26c88b 100644 --- a/src/libstore/include/nix/store/restricted-store.hh +++ b/src/libstore/include/nix/store/restricted-store.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "local-store.hh" +#include "nix/store/local-store.hh" namespace nix { diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index b9e729092..20b25b4ca 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -225,7 +225,7 @@ void Store::queryMissing(const std::vector & targets, ParsedDerivation parsedDrv(*drv); DerivationOptions drvOptions; try { - drvOptions = DerivationOptions::fromParsedDerivation(parsedDrv); + drvOptions = DerivationOptions::fromParsedDerivation(*this, parsedDrv); } catch (Error & e) { e.addTrace({}, "while parsing derivation '%s'", printStorePath(drvPath)); throw; diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 66bf76cac..c4d4ff0d6 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -1,4 +1,5 @@ #include "nix/store/parsed-derivations.hh" +#include "nix/store/derivation-options.hh" #include #include @@ -151,7 +152,10 @@ static nlohmann::json pathInfoToJSON( return jsonList; } -std::optional ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths) +std::optional ParsedDerivation::prepareStructuredAttrs( + Store & store, + const DerivationOptions & drvOptions, + const StorePathSet & inputPaths) { if (!structuredAttrs) return std::nullopt; @@ -164,15 +168,9 @@ std::optional ParsedDerivation::prepareStructuredAttrs(Store & s json["outputs"] = outputs; /* Handle exportReferencesGraph. */ - auto e = json.find("exportReferencesGraph"); - if (e != json.end() && e->is_object()) { - for (auto i = e->begin(); i != e->end(); ++i) { - StorePathSet storePaths; - for (auto & p : *i) - storePaths.insert(store.toStorePath(p.get()).first); - json[i.key()] = pathInfoToJSON(store, - store.exportReferences(storePaths, inputPaths)); - } + for (auto & [key, storePaths] : drvOptions.exportReferencesGraph) { + json[key] = pathInfoToJSON(store, + store.exportReferences(storePaths, inputPaths)); } return json; diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 8adc001f0..b306c3ccd 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -24,7 +24,6 @@ #include "nix/store/restricted-store.hh" #include "nix/store/config.hh" -#include #include #include @@ -973,32 +972,11 @@ void LocalDerivationGoal::startBuilder() /* Handle exportReferencesGraph(), if set. */ if (!parsedDrv->hasStructuredAttrs()) { - /* The `exportReferencesGraph' feature allows the references graph - to be passed to a builder. This attribute should be a list of - pairs [name1 path1 name2 path2 ...]. The references graph of - each `pathN' will be stored in a text file `nameN' in the - temporary build directory. The text files have the format used - by `nix-store --register-validity'. However, the deriver - fields are left empty. */ - auto s = getOr(drv->env, "exportReferencesGraph", ""); - Strings ss = tokenizeString(s); - if (ss.size() % 2 != 0) - throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); - for (Strings::iterator i = ss.begin(); i != ss.end(); ) { - auto fileName = *i++; - static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); - if (!std::regex_match(fileName, regex)) - throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); - - auto storePathS = *i++; - if (!worker.store.isInStore(storePathS)) - throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); - auto storePath = worker.store.toStorePath(storePathS).first; - + for (auto & [fileName, storePathSet] : drvOptions->exportReferencesGraph) { /* Write closure info to . */ writeFile(tmpDir + "/" + fileName, worker.store.makeValidityRegistration( - worker.store.exportReferences({storePath}, inputPaths), false, false)); + worker.store.exportReferences(storePathSet, inputPaths), false, false)); } } @@ -1592,7 +1570,7 @@ void LocalDerivationGoal::initEnv() void LocalDerivationGoal::writeStructuredAttrs() { - if (auto structAttrsJson = parsedDrv->prepareStructuredAttrs(worker.store, inputPaths)) { + if (auto structAttrsJson = parsedDrv->prepareStructuredAttrs(worker.store, *drvOptions, inputPaths)) { auto json = structAttrsJson.value(); nlohmann::json rewritten; for (auto & [i, v] : json["outputs"].get()) { diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index d90630438..e07233356 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -547,7 +547,7 @@ static void main_nix_build(int argc, char * * argv) ParsedDerivation parsedDrv(drv); DerivationOptions drvOptions; try { - drvOptions = DerivationOptions::fromParsedDerivation(parsedDrv); + drvOptions = DerivationOptions::fromParsedDerivation(*store, parsedDrv); } catch (Error & e) { e.addTrace({}, "while parsing derivation '%s'", store->printStorePath(packageInfo.requireDrvPath())); throw; @@ -584,7 +584,7 @@ static void main_nix_build(int argc, char * * argv) for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) accumInputClosure(inputDrv, inputNode); - if (auto structAttrs = parsedDrv.prepareStructuredAttrs(*store, inputs)) { + if (auto structAttrs = parsedDrv.prepareStructuredAttrs(*store, drvOptions, inputs)) { auto json = structAttrs.value(); structuredAttrsRC = writeStructuredAttrsShell(json);