Make store setting and store flags use StoreReference

This commit is contained in:
John Ericson 2024-01-24 11:05:26 -05:00
parent 4a19f4a866
commit a7c841f704
35 changed files with 269 additions and 105 deletions

View File

@ -75,28 +75,32 @@ CopyCommand::CopyCommand()
.longName = "from",
.description = "URL of the source Nix store.",
.labels = {"store-uri"},
.handler = {&srcUri},
.handler = {[&](std::string storeUri) {
srcUri = StoreReference::parse(storeUri);
}},
});
addFlag({
.longName = "to",
.description = "URL of the destination Nix store.",
.labels = {"store-uri"},
.handler = {&dstUri},
.handler = {[&](std::string storeUri) {
dstUri = StoreReference::parse(storeUri);
}},
});
}
ref<Store> CopyCommand::createStore()
{
return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
return !srcUri ? StoreCommand::createStore() : openStore(*srcUri);
}
ref<Store> CopyCommand::getDstStore()
{
if (srcUri.empty() && dstUri.empty())
if (!srcUri && !dstUri)
throw UsageError("you must pass '--from' and/or '--to'");
return dstUri.empty() ? openStore() : openStore(dstUri);
return !dstUri ? openStore() : openStore(*dstUri);
}
EvalCommand::EvalCommand()

View File

@ -62,7 +62,7 @@ private:
*/
struct CopyCommand : virtual StoreCommand
{
std::string srcUri, dstUri;
std::optional<StoreReference> srcUri, dstUri;
CopyCommand();

View File

@ -164,7 +164,9 @@ MixEvalArgs::MixEvalArgs()
)",
.category = category,
.labels = {"store-url"},
.handler = {&evalStoreUrl},
.handler = {[&](std::string s) {
evalStoreUrl = StoreReference::parse(s);
}},
});
}

View File

@ -5,6 +5,7 @@
#include "canon-path.hh"
#include "common-args.hh"
#include "search-path.hh"
#include "store-reference.hh"
#include <filesystem>
@ -25,7 +26,7 @@ struct MixEvalArgs : virtual Args, virtual MixRepair
LookupPath lookupPath;
std::optional<std::string> evalStoreUrl;
std::optional<StoreReference> evalStoreUrl;
private:
struct AutoArgExpr { std::string expr; };

View File

@ -1,3 +1,5 @@
#include <nlohmann/json.hpp>
#include "terminal.hh"
#include "flake.hh"
#include "eval.hh"

View File

@ -182,23 +182,26 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
.pos = state.positions[pos]
});
auto parsedURL = parseURL(*fromStoreUrl);
auto parsedURL = StoreReference::parse(*fromStoreUrl);
if (parsedURL.scheme != "http" &&
parsedURL.scheme != "https" &&
!(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file"))
auto * storeVariant = std::get_if<StoreReference::Specified>(&parsedURL.variant);
if (!storeVariant ||
(storeVariant->scheme != "http" &&
storeVariant->scheme != "https" &&
!(getEnv("_NIX_IN_TEST").has_value() && storeVariant->scheme == "file")))
throw Error({
.msg = HintFmt("'fetchClosure' only supports http:// and https:// stores"),
.pos = state.positions[pos]
});
if (!parsedURL.query.empty())
if (!parsedURL.params.empty())
throw Error({
.msg = HintFmt("'fetchClosure' does not support URL query parameters (in '%s')", *fromStoreUrl),
.pos = state.positions[pos]
});
auto fromStore = openStore(parsedURL.to_string());
auto fromStore = openStore(parsedURL);
if (toPath)
runFetchClosureWithRewrite(state, pos, *fromStore, *fromPath, *toPath, v);

View File

@ -15,6 +15,8 @@
#include <iomanip>
#include <regex>
#include <nlohmann/json.hpp>
namespace nix {
void emitTreeAttrs(

View File

@ -13,6 +13,7 @@
#include "git-utils.hh"
#include "logging.hh"
#include "finally.hh"
#include "json-utils.hh"
#include "fetch-settings.hh"

View File

@ -49,14 +49,15 @@ Store * nix_store_open(nix_c_context * context, const char * uri, const char ***
if (uri_str.empty())
return new Store{nix::openStore()};
if (!params)
return new Store{nix::openStore(uri_str)};
auto base_uri = nix::StoreReference::parse(uri_str);
nix::Store::Params params_map;
if (params) {
for (size_t i = 0; params[i] != nullptr; i++) {
params_map[params[i][0]] = params[i][1];
base_uri.params[params[i][0]] = params[i][1];
}
return new Store{nix::openStore(uri_str, params_map)};
}
return new Store{nix::openStore(std::move(base_uri))};
}
NIXC_CATCH_ERRS_NULL
}

View File

@ -215,22 +215,22 @@ struct ClientSettings
auto & name(i.first);
auto & value(i.second);
auto setSubstituters = [&](Setting<Strings> & res) {
auto setSubstituters = [&](Setting<std::vector<StoreReference>> & res) {
if (name != res.name && res.aliases.count(name) == 0)
return false;
StringSet trusted = settings.trustedSubstituters;
std::set<StoreReference> trusted = settings.trustedSubstituters;
for (auto & s : settings.substituters.get())
trusted.insert(s);
Strings subs;
std::vector<StoreReference> subs;
auto ss = tokenizeString<Strings>(value);
for (auto & s : ss)
if (trusted.count(s))
subs.push_back(s);
else if (!hasSuffix(s, "/") && trusted.count(s + "/"))
subs.push_back(s + "/");
for (auto & s : ss) {
auto ref = StoreReference::parse(s);
if (trusted.count(ref))
subs.push_back(ref);
else
warn("ignoring untrusted substituter '%s', you are not a trusted user.\n"
"Run `man nix.conf` for more information on the `substituters` configuration option.", s);
}
res = subs;
return true;
};

View File

@ -1,3 +1,6 @@
#include <boost/container/small_vector.hpp>
#include <nlohmann/json.hpp>
#include "derivations.hh"
#include "downstream-placeholder.hh"
#include "store-api.hh"
@ -7,8 +10,7 @@
#include "split.hh"
#include "common-protocol.hh"
#include "common-protocol-impl.hh"
#include <boost/container/small_vector.hpp>
#include <nlohmann/json.hpp>
#include "json-utils.hh"
namespace nix {

View File

@ -35,6 +35,10 @@
namespace nix {
DECLARE_CONFIG_SERIALISER(StoreReference)
DECLARE_CONFIG_SERIALISER(std::optional<StoreReference>)
DECLARE_CONFIG_SERIALISER(std::set<StoreReference>)
DECLARE_CONFIG_SERIALISER(std::vector<StoreReference>)
/* The default location of the daemon socket, relative to nixStateDir.
The socket is in a directory to allow you to control access to the
@ -319,6 +323,66 @@ template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::s
});
}
template<> StoreReference BaseSetting<StoreReference>::parse(const std::string & str) const
{
return StoreReference::parse(str);
}
template<> std::string BaseSetting<StoreReference>::to_string() const
{
return value.render();
}
template<> std::optional<StoreReference> BaseSetting<std::optional<StoreReference>>::parse(const std::string & str) const
{
if (str == "")
return std::nullopt;
else
return StoreReference::parse(str);
}
template<> std::string BaseSetting<std::optional<StoreReference>>::to_string() const
{
return value ? value->render() : "";
}
template<> std::set<StoreReference> BaseSetting<std::set<StoreReference>>::parse(const std::string & str) const
{
std::set<StoreReference> res;
for (const auto & s : tokenizeString<std::vector<std::string>>(str))
res.insert(StoreReference::parse(s));
return res;
}
template<> std::string BaseSetting<std::set<StoreReference>>::to_string() const
{
std::set<std::string> strings;
for (const auto & ref : value)
strings.insert(ref.render());
return concatStringsSep(" ", strings);
}
template<> std::vector<StoreReference> BaseSetting<std::vector<StoreReference>>::parse(const std::string & str) const
{
std::vector<StoreReference> res;
for (const auto & s : tokenizeString<std::vector<std::string>>(str))
res.push_back(StoreReference::parse(s));
return res;
}
template<> std::string BaseSetting<std::vector<StoreReference>>::to_string() const
{
std::vector<std::string> strings;
for (const auto & ref : value)
strings.push_back(ref.render());
return concatStringsSep(" ", strings);
}
template class BaseSetting<StoreReference>;
template class BaseSetting<std::optional<StoreReference>>;
template class BaseSetting<std::set<StoreReference>>;
template class BaseSetting<std::vector<StoreReference>>;
unsigned int MaxBuildJobsSetting::parse(const std::string & str) const
{
if (str == "auto") return std::max(1U, std::thread::hardware_concurrency());

View File

@ -5,6 +5,7 @@
#include "config.hh"
#include "environment-variables.hh"
#include "experimental-features.hh"
#include "store-reference.hh"
#include "users.hh"
#include <map>
@ -116,7 +117,7 @@ public:
*/
Path nixDaemonSocketFile;
Setting<std::string> storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store",
Setting<StoreReference> storeUri{this, StoreReference::parse(getEnv("NIX_REMOTE").value_or("auto")), "store",
R"(
The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format)
to use for most operations.
@ -904,9 +905,16 @@ public:
// Don't document the machine-specific default value
false};
Setting<Strings> substituters{
Setting<std::vector<StoreReference>> substituters{
this,
Strings{"https://cache.nixos.org/"},
{
{
.variant = StoreReference::Specified {
.scheme = "https",
.authority = "cache.nixos.org",
},
},
},
"substituters",
R"(
A list of [URLs of Nix stores](@docroot@/store/types/index.md#store-url-format) to be used as substituters, separated by whitespace.
@ -925,7 +933,7 @@ public:
)",
{"binary-caches"}};
Setting<StringSet> trustedSubstituters{
Setting<std::set<StoreReference>> trustedSubstituters{
this, {}, "trusted-substituters",
R"(
A list of [Nix store URLs](@docroot@/store/types/index.md#store-url-format), separated by whitespace.

View File

@ -68,9 +68,9 @@ ref<LegacySSHStore::Connection> LegacySSHStore::openConnection()
Strings command = remoteProgram.get();
command.push_back("--serve");
command.push_back("--write");
if (remoteStore.get() != "") {
if (std::optional rs = remoteStore.get()) {
command.push_back("--store");
command.push_back(remoteStore.get());
command.push_back(rs->render());
}
conn->sshConn = master.startCommand(std::move(command));
conn->to = FdSink(conn->sshConn->in.get());

View File

@ -1,6 +1,7 @@
#include "globals.hh"
#include "nar-info.hh"
#include "store-api.hh"
#include "json-utils.hh"
namespace nix {

View File

@ -18,7 +18,7 @@ struct CommonSSHStoreConfig : virtual StoreConfig
const Setting<bool> compress{this, false, "compress",
"Whether to enable SSH compression."};
const Setting<std::string> remoteStore{this, "", "remote-store",
const Setting<std::optional<StoreReference>> remoteStore{this, std::nullopt, "remote-store",
R"(
[Store URL](@docroot@/store/types/index.md#store-url-format)
to be used on the remote machine. The default is `auto`

View File

@ -228,9 +228,9 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
auto conn = make_ref<Connection>();
Strings command = remoteProgram.get();
command.push_back("--stdio");
if (remoteStore.get() != "") {
if (std::optional rs = remoteStore.get()) {
command.push_back("--store");
command.push_back(remoteStore.get());
command.push_back(rs->render());
}
command.insert(command.end(),
extraRemoteProgramArgs.begin(), extraRemoteProgramArgs.end());

View File

@ -1330,9 +1330,9 @@ std::list<ref<Store>> getDefaultSubstituters()
static auto stores([]() {
std::list<ref<Store>> stores;
StringSet done;
std::set<StoreReference> done;
auto addStore = [&](const std::string & uri) {
auto addStore = [&](const StoreReference & uri) {
if (!done.insert(uri).second) return;
try {
stores.push_back(openStore(uri));

View File

@ -863,13 +863,10 @@ OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * ev
*/
ref<Store> openStore(StoreReference && storeURI);
/**
* Opens the store at `uri`, where `uri` is in the format expected by `StoreReference::parse`
*/
ref<Store> openStore(const std::string & uri = settings.storeUri.get(),
const Store::Params & extraParams = Store::Params());
static inline ref<Store> openStore(const StoreReference & storeURI = settings.storeUri.get())
{
return openStore(StoreReference { storeURI });
}
/**

View File

@ -1,4 +1,5 @@
#include <regex>
#include <nlohmann/json.hpp>
#include "error.hh"
#include "url.hh"
@ -114,3 +115,22 @@ std::pair<std::string, StoreReference::Params> splitUriAndParams(const std::stri
}
}
namespace nlohmann {
using namespace nix;
// TODO support something structured
StoreReference adl_serializer<StoreReference>::from_json(const json & json)
{
auto s = json.get<std::string>();
return StoreReference::parse(s);
}
void adl_serializer<StoreReference>::to_json(json & json, StoreReference ref)
{
json = ref.render();
}
}

View File

@ -4,6 +4,8 @@
#include <variant>
#include "types.hh"
#include "json-impls.hh"
#include "json-avoids-null.hh"
namespace nix {
@ -89,4 +91,14 @@ struct StoreReference
*/
std::pair<std::string, StoreReference::Params> splitUriAndParams(const std::string & uri);
/**
* It is always rendered as a string
*/
template<>
struct json_avoids_null<StoreReference> : std::true_type
{
};
}
JSON_IMPL(StoreReference)

View File

@ -26,7 +26,7 @@ LocalOverlayStore::LocalOverlayStore(const Params & params)
, Store(params)
, LocalFSStore(params)
, LocalStore(params)
, lowerStore(openStore(percentDecode(lowerStoreUri.get())).dynamic_pointer_cast<LocalFSStore>())
, lowerStore(openStore(lowerStoreUri.get()).dynamic_pointer_cast<LocalFSStore>())
{
if (checkMount.get()) {
std::smatch match;

View File

@ -13,7 +13,7 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig
, LocalStoreConfig(params)
{ }
const Setting<std::string> lowerStoreUri{(StoreConfig*) this, "", "lower-store",
const Setting<StoreReference> lowerStoreUri{this, StoreReference{.variant = StoreReference::Auto{}}, "lower-store",
R"(
[Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly).
@ -22,12 +22,12 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig
Must be used as OverlayFS lower layer for this store's store dir.
)"};
const PathSetting upperLayer{(StoreConfig*) this, "", "upper-layer",
const PathSetting upperLayer{this, "", "upper-layer",
R"(
Directory containing the OverlayFS upper layer for this store's store dir.
)"};
Setting<bool> checkMount{(StoreConfig*) this, true, "check-mount",
Setting<bool> checkMount{this, true, "check-mount",
R"(
Check that the overlay filesystem is correctly mounted.
@ -38,7 +38,7 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig
default, but can be disabled if needed.
)"};
const PathSetting remountHook{(StoreConfig*) this, "", "remount-hook",
const PathSetting remountHook{this, "", "remount-hook",
R"(
Script or other executable to run when overlay filesystem needs remounting.

View File

@ -3,7 +3,7 @@
#include "comparator.hh"
#include "error.hh"
#include "json-utils.hh"
#include "json-avoids-null.hh"
#include "types.hh"
namespace nix {

View File

@ -0,0 +1,57 @@
#pragma once
///@file
#include <list>
#include <nlohmann/json_fwd.hpp>
#include "types.hh"
namespace nix {
/**
* For `adl_serializer<std::optional<T>>` below, we need to track what
* types are not already using `null`. Only for them can we use `null`
* to represent `std::nullopt`.
*/
template<typename T>
struct json_avoids_null;
/**
* Handle numbers in default impl
*/
template<typename T>
struct json_avoids_null : std::bool_constant<std::is_integral<T>::value>
{
};
template<>
struct json_avoids_null<std::nullptr_t> : std::false_type
{
};
template<>
struct json_avoids_null<bool> : std::true_type
{
};
template<>
struct json_avoids_null<std::string> : std::true_type
{
};
template<typename T>
struct json_avoids_null<std::vector<T>> : std::true_type
{
};
template<typename T>
struct json_avoids_null<std::list<T>> : std::true_type
{
};
template<typename K, typename V>
struct json_avoids_null<std::map<K, V>> : std::true_type
{
};
}

View File

@ -1,7 +1,7 @@
#pragma once
///@file
#include "nlohmann/json_fwd.hpp"
#include <nlohmann/json_fwd.hpp>
// Following https://github.com/nlohmann/json#how-can-i-use-get-for-non-default-constructiblenon-copyable-types
#define JSON_IMPL(TYPE) \

View File

@ -2,10 +2,9 @@
///@file
#include <nlohmann/json.hpp>
#include <list>
#include <nlohmann/json_fwd.hpp>
#include "types.hh"
#include "json-avoids-null.hh"
namespace nix {
@ -39,38 +38,6 @@ Strings getStringList(const nlohmann::json & value);
StringMap getStringMap(const nlohmann::json & value);
StringSet getStringSet(const nlohmann::json & value);
/**
* For `adl_serializer<std::optional<T>>` below, we need to track what
* types are not already using `null`. Only for them can we use `null`
* to represent `std::nullopt`.
*/
template<typename T>
struct json_avoids_null;
/**
* Handle numbers in default impl
*/
template<typename T>
struct json_avoids_null : std::bool_constant<std::is_integral<T>::value> {};
template<>
struct json_avoids_null<std::nullptr_t> : std::false_type {};
template<>
struct json_avoids_null<bool> : std::true_type {};
template<>
struct json_avoids_null<std::string> : std::true_type {};
template<typename T>
struct json_avoids_null<std::vector<T>> : std::true_type {};
template<typename T>
struct json_avoids_null<std::list<T>> : std::true_type {};
template<typename K, typename V>
struct json_avoids_null<std::map<K, V>> : std::true_type {};
}
namespace nlohmann {

View File

@ -47,7 +47,15 @@ static int main_nix_copy_closure(int argc, char ** argv)
if (sshHost.empty())
throw UsageError("no host name specified");
auto remoteUri = "ssh://" + sshHost + (gzip ? "?compress=true" : "");
StoreReference remoteUri{
.variant =
StoreReference::Specified{
.scheme = "ssh",
.authority = sshHost,
},
.params = gzip ? (StoreReference::Params{{"compress", "true"}}) : (StoreReference::Params{}),
};
auto to = toMode ? openStore(remoteUri) : openStore();
auto from = toMode ? openStore() : openStore(remoteUri);

View File

@ -1006,7 +1006,7 @@ struct CmdFlakeClone : FlakeCommand
struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
{
std::string dstUri;
std::optional<StoreReference> dstUri;
CmdFlakeArchive()
{
@ -1014,7 +1014,9 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
.longName = "to",
.description = "URI of the destination Nix store",
.labels = {"store-uri"},
.handler = {&dstUri}
.handler = {[&](std::string s) {
dstUri = StoreReference::parse(s);
}},
});
}
@ -1075,8 +1077,8 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
traverse(*flake.lockFile.root);
}
if (!dryRun && !dstUri.empty()) {
ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
if (!dryRun && dstUri) {
ref<Store> dstStore = openStore(*dstUri);
copyPaths(*store, *dstStore, sources);
}
}

View File

@ -18,6 +18,7 @@
#include "terminal.hh"
#include "users.hh"
#include "network-proxy.hh"
#include "json-utils.hh"
#include <sys/types.h>
#include <regex>
@ -224,7 +225,7 @@ static void showHelp(std::vector<std::string> subcommand, NixArgs & toplevel)
evalSettings.restrictEval = false;
evalSettings.pureEval = false;
EvalState state({}, openStore("dummy://"));
EvalState state({}, openStore({ .variant = StoreReference::Specified { .scheme = "dummy" } }));
auto vGenerateManpage = state.allocValue();
state.eval(state.parseExprFromString(
@ -400,7 +401,7 @@ void mainWrapped(int argc, char * * argv)
Xp::FetchTree,
};
evalSettings.pureEval = false;
EvalState state({}, openStore("dummy://"));
EvalState state({}, openStore({ .variant = StoreReference::Specified { .scheme = "dummy" } }));
auto res = nlohmann::json::object();
res["builtins"] = ({
auto builtinsJson = nlohmann::json::object();

View File

@ -30,7 +30,7 @@ struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand,
void run(ref<Store> srcStore, StorePaths && storePaths) override
{
auto dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
auto dstStore = !dstUri ? openStore() : openStore(*dstUri);
auto remappings = makeContentAddressed(*srcStore, *dstStore,
StorePathSet(storePaths.begin(), storePaths.end()));

View File

@ -11,7 +11,7 @@ using namespace nix;
struct CmdCopySigs : StorePathsCommand
{
Strings substituterUris;
std::vector<StoreReference> substituterUris;
CmdCopySigs()
{
@ -20,7 +20,9 @@ struct CmdCopySigs : StorePathsCommand
.shortName = 's',
.description = "Copy signatures from the specified store.",
.labels = {"store-uri"},
.handler = {[&](std::string s) { substituterUris.push_back(s); }},
.handler = {[&](std::string s) {
substituterUris.push_back(StoreReference::parse(s));
}},
});
}

View File

@ -234,10 +234,10 @@ static PeerInfo getPeerInfo(int remote)
*/
static ref<Store> openUncachedStore()
{
Store::Params params; // FIXME: get params from somewhere
StoreReference ref = settings.storeUri.get();
// Disable caching since the client already does that.
params["path-info-cache-size"] = "0";
return openStore(settings.storeUri, params);
ref.params["path-info-cache-size"] = "0";
return openStore(std::move(ref));
}
/**

View File

@ -15,7 +15,7 @@ struct CmdVerify : StorePathsCommand
{
bool noContents = false;
bool noTrust = false;
Strings substituterUris;
std::vector<StoreReference> substituterUris;
size_t sigsNeeded = 0;
CmdVerify()
@ -37,7 +37,9 @@ struct CmdVerify : StorePathsCommand
.shortName = 's',
.description = "Use signatures from the specified store.",
.labels = {"store-uri"},
.handler = {[&](std::string s) { substituterUris.push_back(s); }}
.handler = {[&](std::string s) {
substituterUris.push_back(StoreReference::parse(s));
}}
});
addFlag({

View File

@ -16,7 +16,12 @@ class LibStoreTest : public virtual ::testing::Test {
protected:
LibStoreTest()
: store(openStore("dummy://"))
: store(openStore({
.variant = StoreReference::Specified {
.scheme = "dummy",
.authority = "",
},
}))
{ }
ref<Store> store;