New store settings system

Motivation:

See the linked issues for details.

The most notable user-relevant bits are:

- This cleans up the `MountedSSHStore`: decomposed into its orthogonal parts

- This brings us pretty close to being able to then implement a JSON-based config.
   - Store query parameters can be JSON
   - Stores can entirely be specified via JSON objects, but this is not yet hooked up to anything.

Also behind the scenes have these benefits:

1. The docs are moved out of the headers, good for less rebuilding when they changes
2. Stores are always constructed from store configs
3. Use JSON, avoid custom serializers

Context:

Part of #11106
Part of #10766
This commit is contained in:
John Ericson 2025-03-17 11:29:06 -04:00
parent 71567373b6
commit 668c465a08
106 changed files with 2797 additions and 1234 deletions

View File

@ -19,7 +19,6 @@ in
prefix,
inlineHTML ? true,
}:
settingsInfo:
let
@ -27,11 +26,25 @@ let
prefix: setting:
{
description,
documentDefault,
defaultValue,
aliases,
value,
experimentalFeature,
# Whether we document the default, because it is machine agostic,
# or don't because because it is machine-specific.
documentDefault ? true,
# The default value is JSON for new-style config, rather than then
# a string or boolean, for old-style config.
json ? false,
defaultValue ? null,
subSettings ? null,
aliases ? [ ],
# The current value for this setting. Purposefully unused.
value ? null,
}:
let
result = squash ''
@ -50,7 +63,7 @@ let
${description}
**Default:** ${showDefault documentDefault defaultValue}
${showDefaultOrSubSettings}
${showAliases aliases}
'';
@ -72,9 +85,24 @@ let
> ```
'';
showDefaultOrSubSettings =
if !isAttrs subSettings then
# No subsettings, instead single setting. Show the default value.
''
**Default:** ${showDefault}
''
else
# Indent the nested sub-settings, and append the outer setting name onto the prefix
indent " " ''
**Nullable sub-settings**: ${if subSettings.nullable then "true" else "false"}
${builtins.trace prefix (showSettings "${prefix}-${setting}" subSettings.map)}
'';
showDefault =
documentDefault: defaultValue:
if documentDefault then
if json then
"`${builtins.toJSON defaultValue}`"
else
# a StringMap value type is specified as a string, but
# this shows the value type. The empty stringmap is `null` in
# JSON, but that converts to `{ }` here.
@ -95,5 +123,7 @@ let
in
result;
showSettings =
prefix: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting prefix) settingsInfo));
in
concatStrings (attrValues (mapAttrs (showSetting prefix) settingsInfo))
showSettings prefix

View File

@ -33,6 +33,7 @@ let
{
settings,
doc,
uri-schemes,
experimentalFeature,
}:
let

View File

@ -16,7 +16,7 @@
#include "nix/store/globals.hh"
#include "nix/util/serialise.hh"
#include "nix/store/build-result.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/util/strings.hh"
#include "nix/store/derivations.hh"
#include "nix/store/local-store.hh"
@ -44,7 +44,7 @@ static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
static bool allSupportedLocally(Store & store, const std::set<std::string>& requiredFeatures) {
for (auto & feature : requiredFeatures)
if (!store.systemFeatures.get().count(feature)) return false;
if (!store.config.systemFeatures.get().count(feature)) return false;
return true;
}
@ -85,7 +85,7 @@ static int main_build_remote(int argc, char * * argv)
that gets cleared on reboot, but it wouldn't work on macOS. */
auto currentLoadName = "/current-load";
if (auto localStore = store.dynamic_pointer_cast<LocalFSStore>())
currentLoad = std::string { localStore->stateDir } + currentLoadName;
currentLoad = std::string { localStore->config.stateDir } + currentLoadName;
else
currentLoad = settings.nixStateDir + currentLoadName;

View File

@ -3,7 +3,7 @@
#include "nix/cmd/command.hh"
#include "nix/cmd/markdown.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/store/local-fs-store.hh"
#include "nix/store/derivations.hh"
#include "nix/expr/nixexpr.hh"

View File

@ -9,7 +9,7 @@
#include "nix/fetchers/registry.hh"
#include "nix/flake/flakeref.hh"
#include "nix/flake/settings.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/cmd/command.hh"
#include "nix/fetchers/tarball.hh"
#include "nix/fetchers/fetch-to-store.hh"

View File

@ -18,7 +18,7 @@ extern char ** savedArgv;
class EvalState;
struct Pos;
class Store;
class LocalFSStore;
struct LocalFSStore;
static constexpr Command::Category catHelp = -1;
static constexpr Command::Category catSecondary = 100;

View File

@ -12,7 +12,7 @@
#include "nix/expr/eval-settings.hh"
#include "nix/expr/attr-path.hh"
#include "nix/util/signals.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/store/log-store.hh"
#include "nix/cmd/common-eval-args.hh"
#include "nix/expr/get-drvs.hh"

View File

@ -1,5 +1,5 @@
#include "nix/expr/primops.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/store/realisation.hh"
#include "nix/store/make-content-addressed.hh"
#include "nix/util/url.hh"

View File

@ -5,6 +5,7 @@
#include "nix/store/path.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/store/build-result.hh"
#include "nix/store/globals.hh"
@ -42,7 +43,7 @@ Store * nix_store_open(nix_c_context * context, const char * uri, const char ***
if (!params)
return new Store{nix::openStore(uri_str)};
nix::Store::Params params_map;
nix::StoreReference::Params params_map;
for (size_t i = 0; params[i] != nullptr; i++) {
params_map[params[i][0]] = params[i][1];
}

View File

@ -5,6 +5,7 @@
#include <gmock/gmock.h>
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
namespace nix {

View File

@ -0,0 +1,3 @@
{
"scheme": "auto"
}

View File

@ -0,0 +1,4 @@
{
"root": "/foo/bar/baz",
"scheme": "auto"
}

View File

@ -0,0 +1,5 @@
{
"authority": "",
"root": "/foo/bar/baz",
"scheme": "local"
}

View File

@ -0,0 +1,5 @@
{
"authority": "/foo/bar/baz",
"scheme": "local",
"trusted": true
}

View File

@ -0,0 +1,4 @@
{
"authority": "localhost",
"scheme": "ssh"
}

View File

@ -0,0 +1,6 @@
{
"authority": "",
"max-connections": 7,
"scheme": "unix",
"trusted": true
}

View File

@ -0,0 +1,20 @@
#include <gtest/gtest.h>
#include "nix/store/dummy-store.hh"
#include "nix/store/globals.hh"
namespace nix {
TEST(DummyStore, constructConfig)
{
DummyStoreConfig config{"dummy", "", {}};
EXPECT_EQ(config.storeDir, settings.nixStore);
}
TEST(DummyStore, constructConfigNoAuthority)
{
EXPECT_THROW(DummyStoreConfig("dummy", "not-allowed", {}), UsageError);
}
} // namespace nix

View File

@ -9,11 +9,13 @@ TEST(LegacySSHStore, constructConfig)
LegacySSHStoreConfig config{
"ssh",
"localhost",
StoreConfig::Params{
StoreReference::Params{
{
"remote-program",
// TODO #11106, no more split on space
"foo bar",
{
"foo",
"bar",
},
},
}};
EXPECT_EQ(

View File

@ -1,9 +1,6 @@
// FIXME: Odd failures for templates that are causing the PR to break
// for now with discussion with @Ericson2314 to comment out.
#if 0
# include <gtest/gtest.h>
#include <gtest/gtest.h>
# include "nix/store/local-overlay-store.hh"
#include "nix/store/local-overlay-store.hh"
namespace nix {
@ -31,4 +28,3 @@ TEST(LocalOverlayStore, constructConfig_rootPath)
}
} // namespace nix
#endif

View File

@ -1,15 +1,6 @@
// FIXME: Odd failures for templates that are causing the PR to break
// for now with discussion with @Ericson2314 to comment out.
#if 0
# include <gtest/gtest.h>
#include <gtest/gtest.h>
# include "nix/store/local-store.hh"
// Needed for template specialisations. This is not good! When we
// overhaul how store configs work, this should be fixed.
# include "nix/util/args.hh"
# include "nix/util/config-impl.hh"
# include "nix/util/abstract-setting-to-json.hh"
#include "nix/store/local-store.hh"
namespace nix {
@ -37,4 +28,3 @@ TEST(LocalStore, constructConfig_rootPath)
}
} // namespace nix
#endif

View File

@ -58,6 +58,7 @@ sources = files(
'derivation-advanced-attrs.cc',
'derivation.cc',
'derived-path.cc',
'dummy-store.cc',
'downstream-placeholder.cc',
'http-binary-cache-store.cc',
'legacy-ssh-store.cc',

View File

@ -97,7 +97,7 @@ TEST_F(nix_api_util_context, nix_store_open_dummy)
nix_libstore_init(ctx);
Store * store = nix_store_open(ctx, "dummy://", nullptr);
ASSERT_EQ(NIX_OK, ctx->last_err_code);
ASSERT_STREQ("dummy", store->ptr->getUri().c_str());
ASSERT_STREQ("dummy://", store->ptr->getUri().c_str());
std::string str;
nix_store_get_version(ctx, store, OBSERVE_STRING(str));

View File

@ -1,22 +1,21 @@
// FIXME: Odd failures for templates that are causing the PR to break
// for now with discussion with @Ericson2314 to comment out.
#if 0
# include <gtest/gtest.h>
#include <gtest/gtest.h>
# include "nix/store/ssh-store.hh"
#include "nix/store/ssh-store.hh"
namespace nix {
TEST(SSHStore, constructConfig)
{
SSHStoreConfig config{
"ssh",
"ssh-ng",
"localhost",
StoreConfig::Params{
StoreReference::Params{
{
"remote-program",
// TODO #11106, no more split on space
"foo bar",
{
"foo",
"bar",
},
},
},
};
@ -31,16 +30,26 @@ TEST(SSHStore, constructConfig)
TEST(MountedSSHStore, constructConfig)
{
MountedSSHStoreConfig config{
"mounted-ssh",
ExperimentalFeatureSettings mockXpSettings;
mockXpSettings.set("experimental-features", "mounted-ssh-store");
SSHStoreConfig config{
"ssh-ng",
"localhost",
StoreConfig::Params{
StoreReference::Params{
{
"remote-program",
// TODO #11106, no more split on space
"foo bar",
{
"foo",
"bar",
},
},
{
"mounted",
nlohmann::json::object_t{},
},
},
mockXpSettings,
};
EXPECT_EQ(
@ -49,7 +58,48 @@ TEST(MountedSSHStore, constructConfig)
"foo",
"bar",
}));
ASSERT_TRUE(config.mounted);
EXPECT_EQ(config.mounted->realStoreDir, "/nix/store");
}
TEST(MountedSSHStore, constructConfigWithFunnyRealStoreDir)
{
ExperimentalFeatureSettings mockXpSettings;
mockXpSettings.set("experimental-features", "mounted-ssh-store");
SSHStoreConfig config{
"ssh-ng",
"localhost",
StoreReference::Params{
{
"remote-program",
{
"foo",
"bar",
},
},
{
"mounted",
nlohmann::json::object_t{
{"real", "/foo/bar"},
},
},
},
mockXpSettings,
};
EXPECT_EQ(
config.remoteProgram.get(),
(Strings{
"foo",
"bar",
}));
ASSERT_TRUE(config.mounted);
EXPECT_EQ(config.mounted->realStoreDir, "/foo/bar");
}
}
#endif

View File

@ -17,14 +17,14 @@ class StoreReferenceTest : public CharacterizationTest, public LibStoreTest
std::filesystem::path goldenMaster(PathView testStem) const override
{
return unitTestData / (testStem + ".txt");
return unitTestData / testStem;
}
};
#define URI_TEST_READ(STEM, OBJ) \
TEST_F(StoreReferenceTest, PathInfo_##STEM##_from_uri) \
{ \
readTest(#STEM, ([&](const auto & encoded) { \
readTest(#STEM ".txt", ([&](const auto & encoded) { \
StoreReference expected = OBJ; \
auto got = StoreReference::parse(encoded); \
ASSERT_EQ(got, expected); \
@ -35,7 +35,7 @@ class StoreReferenceTest : public CharacterizationTest, public LibStoreTest
TEST_F(StoreReferenceTest, PathInfo_##STEM##_to_uri) \
{ \
writeTest( \
#STEM, \
#STEM ".txt", \
[&]() -> StoreReference { return OBJ; }, \
[](const auto & file) { return StoreReference::parse(readFile(file)); }, \
[](const auto & file, const auto & got) { return writeFile(file, got.render()); }); \
@ -45,14 +45,43 @@ class StoreReferenceTest : public CharacterizationTest, public LibStoreTest
URI_TEST_READ(STEM, OBJ) \
URI_TEST_WRITE(STEM, OBJ)
URI_TEST(
#define JSON_TEST_READ(STEM, OBJ) \
TEST_F(StoreReferenceTest, PathInfo_##STEM##_from_json) \
{ \
readTest(#STEM ".json", ([&](const auto & encoded_) { \
auto encoded = json::parse(encoded_); \
StoreReference expected = OBJ; \
StoreReference got = encoded; \
ASSERT_EQ(got, expected); \
})); \
}
#define JSON_TEST_WRITE(STEM, OBJ) \
TEST_F(StoreReferenceTest, PathInfo_##STEM##_to_json) \
{ \
writeTest( \
#STEM ".json", \
[&]() -> StoreReference { return OBJ; }, \
[](const auto & file) -> StoreReference { return json::parse(readFile(file)); }, \
[](const auto & file, const auto & got) { return writeFile(file, json(got).dump(2) + "\n"); }); \
}
#define JSON_TEST(STEM, OBJ) \
JSON_TEST_READ(STEM, OBJ) \
JSON_TEST_WRITE(STEM, OBJ)
#define BOTH_FORMATS_TEST(STEM, OBJ) \
URI_TEST(STEM, OBJ) \
JSON_TEST(STEM, OBJ)
BOTH_FORMATS_TEST(
auto,
(StoreReference{
.variant = StoreReference::Auto{},
.params = {},
}))
URI_TEST(
BOTH_FORMATS_TEST(
auto_param,
(StoreReference{
.variant = StoreReference::Auto{},
@ -81,13 +110,13 @@ static StoreReference localExample_2{
},
.params =
{
{"trusted", "true"},
{"trusted", true},
},
};
URI_TEST(local_1, localExample_1)
BOTH_FORMATS_TEST(local_1, localExample_1)
URI_TEST(local_2, localExample_2)
BOTH_FORMATS_TEST(local_2, localExample_2)
URI_TEST_READ(local_shorthand_1, localExample_1)
@ -100,16 +129,16 @@ static StoreReference unixExample{
},
.params =
{
{"max-connections", "7"},
{"trusted", "true"},
{"max-connections", 7},
{"trusted", true},
},
};
URI_TEST(unix, unixExample)
BOTH_FORMATS_TEST(unix, unixExample)
URI_TEST_READ(unix_shorthand, unixExample)
URI_TEST(
BOTH_FORMATS_TEST(
ssh,
(StoreReference{
.variant =

View File

@ -1,9 +1,6 @@
// FIXME: Odd failures for templates that are causing the PR to break
// for now with discussion with @Ericson2314 to comment out.
#if 0
# include <gtest/gtest.h>
#include <gtest/gtest.h>
# include "nix/store/uds-remote-store.hh"
#include "nix/store/uds-remote-store.hh"
namespace nix {
@ -20,4 +17,3 @@ TEST(UDSRemoteStore, constructConfigWrongScheme)
}
} // namespace nix
#endif

View File

@ -13,6 +13,7 @@
#include "nix/util/callback.hh"
#include "nix/util/signals.hh"
#include "nix/util/archive.hh"
#include "nix/store/config-parse-impl.hh"
#include <chrono>
#include <future>
@ -24,17 +25,104 @@
namespace nix {
BinaryCacheStore::BinaryCacheStore(const Params & params)
: BinaryCacheStoreConfig(params)
, Store(params)
constexpr static const BinaryCacheStoreConfigT<config::SettingInfo> binaryCacheStoreConfigDescriptions = {
.compression = {
.name = "compression",
.description = "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`).",
},
.writeNARListing = {
.name = "write-nar-listing",
.description = "Whether to write a JSON file that lists the files in each NAR.",
},
.writeDebugInfo = {
.name = "index-debug-info",
.description = R"(
Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to
fetch debug info on demand
)",
},
.secretKeyFile{
.name = "secret-key",
.description = "Path to the secret key used to sign the binary cache.",
},
.localNarCache{
.name = "local-nar-cache",
.description = "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`.",
},
.parallelCompression{
.name = "parallel-compression",
.description = "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`.",
},
.compressionLevel{
.name = "compression-level",
.description = R"(
The *preset level* to be used when compressing NARs.
The meaning and accepted values depend on the compression method selected.
`-1` specifies that the default compression level should be used.
)",
},
};
#define BINARY_CACHE_STORE_CONFIG_FIELDS(X) \
X(compression), \
X(writeNARListing), \
X(writeDebugInfo), \
X(secretKeyFile), \
X(localNarCache), \
X(parallelCompression), \
X(compressionLevel),
MAKE_PARSE(BinaryCacheStoreConfig, binaryCacheStoreConfig, BINARY_CACHE_STORE_CONFIG_FIELDS)
static BinaryCacheStoreConfigT<config::JustValue> binaryCacheStoreConfigDefaults()
{
if (secretKeyFile != "")
return {
.compression = {"xz"},
.writeNARListing = {false},
.writeDebugInfo = {false},
.secretKeyFile = {""},
.localNarCache = {""},
.parallelCompression = {false},
.compressionLevel = {-1},
};
}
MAKE_APPLY_PARSE(BinaryCacheStoreConfig, binaryCacheStoreConfig, BINARY_CACHE_STORE_CONFIG_FIELDS)
BinaryCacheStore::Config::BinaryCacheStoreConfig(
const Store::Config & storeConfig,
const StoreReference::Params & params)
: BinaryCacheStoreConfigT<config::JustValue>{binaryCacheStoreConfigApplyParse(params)}
, storeConfig{storeConfig}
{
}
config::SettingDescriptionMap BinaryCacheStoreConfig::descriptions()
{
constexpr auto & descriptions = binaryCacheStoreConfigDescriptions;
auto defaults = binaryCacheStoreConfigDefaults();
return {
BINARY_CACHE_STORE_CONFIG_FIELDS(DESC_ROW)
};
}
BinaryCacheStore::BinaryCacheStore(const Config & config)
: Store{config.storeConfig}
, config{config}
{
if (config.secretKeyFile != "")
signer = std::make_unique<LocalSigner>(
SecretKey { readFile(secretKeyFile) });
SecretKey { readFile(config.secretKeyFile) });
StringSink sink;
sink << narVersionMagic1;
narMagic = sink.s;
// Want to call this but cannot, because virtual function lookup is
// disabled in a constructor. It is thus left to instances to call
// it instead.
//init();
}
void BinaryCacheStore::init()
@ -53,9 +141,11 @@ void BinaryCacheStore::init()
throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'",
getUri(), value, storeDir);
} else if (name == "WantMassQuery") {
wantMassQuery.setDefault(value == "1");
resolvedSubstConfig.wantMassQuery.value =
config.storeConfig.wantMassQuery.optValue.value_or(value == "1");
} else if (name == "Priority") {
priority.setDefault(std::stoi(value));
resolvedSubstConfig.priority.value =
config.storeConfig.priority.optValue.value_or(std::stoi(value));
}
}
}
@ -147,7 +237,11 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
{
FdSink fileSink(fdTemp.get());
TeeSink teeSinkCompressed { fileSink, fileHashSink };
auto compressionSink = makeCompressionSink(compression, teeSinkCompressed, parallelCompression, compressionLevel);
auto compressionSink = makeCompressionSink(
config.compression,
teeSinkCompressed,
config.parallelCompression,
config.compressionLevel);
TeeSink teeSinkUncompressed { *compressionSink, narHashSink };
TeeSource teeSource { narSource, teeSinkUncompressed };
narAccessor = makeNarAccessor(teeSource);
@ -159,17 +253,17 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
auto info = mkInfo(narHashSink.finish());
auto narInfo = make_ref<NarInfo>(info);
narInfo->compression = compression;
narInfo->compression = config.compression;
auto [fileHash, fileSize] = fileHashSink.finish();
narInfo->fileHash = fileHash;
narInfo->fileSize = fileSize;
narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Nix32, false) + ".nar"
+ (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" :
compression == "zstd" ? ".zst" :
compression == "lzip" ? ".lzip" :
compression == "lz4" ? ".lz4" :
compression == "br" ? ".br" :
+ (config.compression == "xz" ? ".xz" :
config.compression == "bzip2" ? ".bz2" :
config.compression == "zstd" ? ".zst" :
config.compression == "lzip" ? ".lzip" :
config.compression == "lz4" ? ".lz4" :
config.compression == "br" ? ".br" :
"");
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
@ -191,7 +285,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
/* Optionally write a JSON file containing a listing of the
contents of the NAR. */
if (writeNARListing) {
if (config.writeNARListing) {
nlohmann::json j = {
{"version", 1},
{"root", listNar(ref<SourceAccessor>(narAccessor), CanonPath::root, true)},
@ -203,7 +297,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
/* Optionally maintain an index of DWARF debug info files
consisting of JSON files named 'debuginfo/<build-id>' that
specify the NAR file and member containing the debug info. */
if (writeDebugInfo) {
if (config.writeDebugInfo) {
CanonPath buildIdDir("lib/debug/.build-id");
@ -515,7 +609,7 @@ void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
ref<SourceAccessor> BinaryCacheStore::getFSAccessor(bool requireValidPath)
{
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), requireValidPath, localNarCache);
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), requireValidPath, config.localNarCache);
}
void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs)

View File

@ -1175,7 +1175,7 @@ Path DerivationGoal::openLogFile()
/* Create a log file. */
Path logDir;
if (auto localStore = dynamic_cast<LocalStore *>(&worker.store))
logDir = localStore->logDir;
logDir = localStore->config->logDir;
else
logDir = settings.nixLogDir;
Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, baseName.substr(0, 2));

View File

@ -3,6 +3,7 @@
#include "nix/store/build/worker.hh"
#include "nix/store/build/substitution-goal.hh"
#include "nix/util/callback.hh"
#include "nix/store/store-open.hh"
namespace nix {

View File

@ -1,4 +1,5 @@
#include "nix/store/build/worker.hh"
#include "nix/store/store-open.hh"
#include "nix/store/build/substitution-goal.hh"
#include "nix/store/nar-info.hh"
#include "nix/util/finally.hh"
@ -121,7 +122,7 @@ Goal::Co PathSubstitutionGoal::init()
/* Bail out early if this substituter lacks a valid
signature. LocalStore::addToStore() also checks for this, but
only after we've downloaded the path. */
if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info))
if (!sub->config.isTrusted && worker.store.pathInfoIsUntrusted(*info))
{
warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'",
worker.store.printStorePath(storePath), sub->getUri());
@ -215,7 +216,7 @@ Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref<Store> sub,
PushActivity pact(act.id);
copyStorePath(*sub, worker.store,
subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
subPath, repair, sub->config.isTrusted ? NoCheckSigs : CheckSigs);
promise.set_value();
} catch (...) {

View File

@ -2,9 +2,56 @@
#include "nix/store/common-ssh-store-config.hh"
#include "nix/store/ssh.hh"
#include "nix/store/config-parse-impl.hh"
namespace nix {
constexpr static const CommonSSHStoreConfigT<config::SettingInfo> commonSSHStoreConfigDescriptions = {
.sshKey{
.name = "ssh-key",
.description = "Path to the SSH private key used to authenticate to the remote machine.",
},
.sshPublicHostKey{
.name = "base64-ssh-public-host-key",
.description = "The public host key of the remote machine.",
},
.compress{
.name = "compress",
.description = "Whether to enable SSH compression.",
},
.remoteStore{
.name = "remote-store",
.description = R"(
[Store URL](@docroot@/store/types/index.md#store-url-format)
to be used on the remote machine. The default is `auto`
(i.e. use the Nix daemon or `/nix/store` directly).
)",
},
};
#define COMMON_SSH_STORE_CONFIG_FIELDS(X) X(sshKey), X(sshPublicHostKey), X(compress), X(remoteStore),
MAKE_PARSE(CommonSSHStoreConfig, commonSSHStoreConfig, COMMON_SSH_STORE_CONFIG_FIELDS)
static CommonSSHStoreConfigT<config::JustValue> commonSSHStoreConfigDefaults()
{
return {
.sshKey = {""},
.sshPublicHostKey = {""},
.compress = {false},
.remoteStore = {""},
};
}
MAKE_APPLY_PARSE(CommonSSHStoreConfig, commonSSHStoreConfig, COMMON_SSH_STORE_CONFIG_FIELDS)
config::SettingDescriptionMap CommonSSHStoreConfig::descriptions()
{
constexpr auto & descriptions = commonSSHStoreConfigDescriptions;
auto defaults = commonSSHStoreConfigDefaults();
return {COMMON_SSH_STORE_CONFIG_FIELDS(DESC_ROW)};
}
static std::string extractConnStr(std::string_view scheme, std::string_view _connStr)
{
if (_connStr.empty())
@ -22,13 +69,14 @@ static std::string extractConnStr(std::string_view scheme, std::string_view _con
return connStr;
}
CommonSSHStoreConfig::CommonSSHStoreConfig(std::string_view scheme, std::string_view host, const Params & params)
: StoreConfig(params)
, host(extractConnStr(scheme, host))
CommonSSHStoreConfig::CommonSSHStoreConfig(
std::string_view scheme, std::string_view host, const StoreReference::Params & params)
: CommonSSHStoreConfigT<config::JustValue>{commonSSHStoreConfigApplyParse(params)}
, host{extractConnStr(scheme, host)}
{
}
SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD)
SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD) const
{
return {
host,

View File

@ -0,0 +1,71 @@
#include <nlohmann/json.hpp>
#include "nix/store/config-parse.hh"
#include "nix/util/json-utils.hh"
#include "nix/util/util.hh"
namespace nix::config {
};
namespace nlohmann {
using namespace nix::config;
SettingDescription adl_serializer<SettingDescription>::from_json(const json & json)
{
auto & obj = getObject(json);
return {
.description = getString(valueAt(obj, "description")),
.experimentalFeature = valueAt(obj, "experimentalFeature").get<std::optional<Xp>>(),
.info = [&]() -> decltype(SettingDescription::info) {
if (auto documentDefault = optionalValueAt(obj, "documentDefault")) {
return SettingDescription::Single{
.defaultValue = *documentDefault ? (std::optional<nlohmann::json>{valueAt(obj, "defaultValue")})
: (std::optional<nlohmann::json>{}),
};
} else {
auto & subObj = getObject(valueAt(obj, "subSettings"));
return SettingDescription::Sub{
.nullable = valueAt(subObj, "nullable"),
.map = valueAt(subObj, "map"),
};
}
}(),
};
}
void adl_serializer<SettingDescription>::to_json(json & obj, SettingDescription sd)
{
obj.emplace("description", sd.description);
// obj.emplace("aliases", sd.aliases);
obj.emplace("experimentalFeature", sd.experimentalFeature);
std::visit(
overloaded{
[&](const SettingDescription::Single & single) {
// Indicate the default value is JSON, rather than a legacy setting
// boolean or string.
//
// TODO remove if we no longer have the legacy setting system / the
// code handling doc rendering of the settings is decoupled.
obj.emplace("json", true);
// Cannot just use `null` because the default value might itself be
// `null`.
obj.emplace("documentDefault", single.defaultValue.has_value());
if (single.defaultValue.has_value())
obj.emplace("defaultValue", *single.defaultValue);
},
[&](const SettingDescription::Sub & sub) {
json subJson;
subJson.emplace("nullable", sub.nullable);
subJson.emplace("map", sub.map);
obj.emplace("subSettings", std::move(subJson));
},
},
sd.info);
}
}

View File

@ -160,7 +160,7 @@ bool DerivationOptions::canBuildLocally(Store & localStore, const BasicDerivatio
return false;
for (auto & feature : getRequiredSystemFeatures(drv))
if (!localStore.systemFeatures.get().count(feature))
if (!localStore.config.systemFeatures.get().count(feature))
return false;
return true;

View File

@ -1,47 +1,39 @@
#include "nix/store/store-api.hh"
#include "nix/store/dummy-store.hh"
#include "nix/store/store-registration.hh"
#include "nix/util/callback.hh"
namespace nix {
struct DummyStoreConfig : virtual StoreConfig {
using StoreConfig::StoreConfig;
DummyStoreConfig(std::string_view scheme, std::string_view authority, const Params & params)
: StoreConfig(params)
{
if (!authority.empty())
throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority);
}
const std::string name() override { return "Dummy Store"; }
std::string doc() override
{
return
#include "dummy-store.md"
;
}
static std::set<std::string> uriSchemes() {
return {"dummy"};
}
};
struct DummyStore : public virtual DummyStoreConfig, public virtual Store
DummyStoreConfig::DummyStoreConfig(
std::string_view scheme, std::string_view authority, const StoreReference::Params & params)
: StoreConfig{params}
{
DummyStore(std::string_view scheme, std::string_view authority, const Params & params)
: StoreConfig(params)
, DummyStoreConfig(scheme, authority, params)
, Store(params)
{ }
if (!authority.empty())
throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority);
}
DummyStore(const Params & params)
: DummyStore("dummy", "", params)
std::string DummyStoreConfig::doc()
{
return
#include "dummy-store.md"
;
}
struct DummyStore : virtual Store
{
using Config = DummyStoreConfig;
ref<const Config> config;
DummyStore(ref<const Config> config)
: Store{*config}
, config(config)
{ }
std::string getUri() override
{
return *uriSchemes().begin();
return *Config::uriSchemes().begin() + "://";
}
void queryPathInfoUncached(const StorePath & path,
@ -88,6 +80,11 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
}
};
static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore;
ref<Store> DummyStore::Config::openStore() const
{
return make_ref<DummyStore>(ref{shared_from_this()});
}
static RegisterStoreImplementation<DummyStore::Config> regDummyStore;
}

View File

@ -745,7 +745,7 @@ struct curlFileTransfer : public FileTransfer
}
#if NIX_WITH_S3_SUPPORT
std::tuple<std::string, std::string, Store::Params> parseS3Uri(std::string uri)
std::tuple<std::string, std::string, StoreReference::Params> parseS3Uri(std::string uri)
{
auto [path, params] = splitUriAndParams(uri);

View File

@ -43,7 +43,7 @@ static std::string gcRootsDir = "gcroots";
void LocalStore::addIndirectRoot(const Path & path)
{
std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Nix32, false);
Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash));
Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", config->stateDir, gcRootsDir, hash));
makeSymlink(realRoot, path);
}
@ -82,7 +82,7 @@ void LocalStore::createTempRootsFile()
void LocalStore::addTempRoot(const StorePath & path)
{
if (readOnly) {
if (config->readOnly) {
debug("Read-only store doesn't support creating lock files for temp roots, but nothing can be deleted anyways.");
return;
}
@ -109,7 +109,7 @@ void LocalStore::addTempRoot(const StorePath & path)
auto fdRootsSocket(_fdRootsSocket.lock());
if (!*fdRootsSocket) {
auto socketPath = stateDir.get() + gcSocketPath;
auto socketPath = config->stateDir.get() + gcSocketPath;
debug("connecting to '%s'", socketPath);
*fdRootsSocket = createUnixDomainSocket();
try {
@ -247,7 +247,7 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R
else {
target = absPath(target, dirOf(path));
if (!pathExists(target)) {
if (isInDir(path, std::filesystem::path{stateDir.get()} / gcRootsDir / "auto")) {
if (isInDir(path, std::filesystem::path{config->stateDir.get()} / gcRootsDir / "auto")) {
printInfo("removing stale link from '%1%' to '%2%'", path, target);
unlink(path.c_str());
}
@ -288,8 +288,8 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R
void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
{
/* Process direct roots in {gcroots,profiles}. */
findRoots(stateDir + "/" + gcRootsDir, std::filesystem::file_type::unknown, roots);
findRoots(stateDir + "/profiles", std::filesystem::file_type::unknown, roots);
findRoots(config->stateDir + "/" + gcRootsDir, std::filesystem::file_type::unknown, roots);
findRoots(config->stateDir + "/profiles", std::filesystem::file_type::unknown, roots);
/* Add additional roots returned by different platforms-specific
heuristics. This is typically used to add running programs to
@ -498,7 +498,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
readFile(*p);
/* Start the server for receiving new roots. */
auto socketPath = stateDir.get() + gcSocketPath;
auto socketPath = config->stateDir.get() + gcSocketPath;
createDirs(dirOf(socketPath));
auto fdServer = createUnixDomainSocket(socketPath, 0666);
@ -635,7 +635,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
auto deleteFromStore = [&](std::string_view baseName)
{
Path path = storeDir + "/" + std::string(baseName);
Path realPath = realStoreDir + "/" + std::string(baseName);
Path realPath = config->realStoreDir + "/" + std::string(baseName);
/* There may be temp directories in the store that are still in use
by another process. We need to be sure that we can acquire an
@ -804,8 +804,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
printInfo("determining live/dead paths...");
try {
AutoCloseDir dir(opendir(realStoreDir.get().c_str()));
if (!dir) throw SysError("opening directory '%1%'", realStoreDir);
AutoCloseDir dir(opendir(config->realStoreDir.get().c_str()));
if (!dir) throw SysError("opening directory '%1%'", config->realStoreDir);
/* Read the store and delete all paths that are invalid or
unreachable. We don't use readDirectory() here so that
@ -907,8 +907,8 @@ void LocalStore::autoGC(bool sync)
return std::stoll(readFile(*fakeFreeSpaceFile));
struct statvfs st;
if (statvfs(realStoreDir.get().c_str(), &st))
throw SysError("getting filesystem info about '%s'", realStoreDir);
if (statvfs(config->realStoreDir.get().c_str(), &st))
throw SysError("getting filesystem info about '%s'", config->realStoreDir);
return (uint64_t) st.f_bavail * st.f_frsize;
};

View File

@ -3,18 +3,37 @@
#include "nix/store/globals.hh"
#include "nix/store/nar-info-disk-cache.hh"
#include "nix/util/callback.hh"
#include "nix/store/store-registration.hh"
namespace nix {
config::SettingDescriptionMap HttpBinaryCacheStoreConfig::descriptions()
{
config::SettingDescriptionMap ret;
ret.merge(StoreConfig::descriptions());
ret.merge(BinaryCacheStoreConfig::descriptions());
return ret;
}
MakeError(UploadToHTTP, Error);
std::set<std::string> HttpBinaryCacheStoreConfig::uriSchemes()
{
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
auto ret = std::set<std::string>({"http", "https"});
if (forceHttp)
ret.insert("file");
return ret;
}
HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig(
std::string_view scheme,
std::string_view _cacheUri,
const Params & params)
: StoreConfig(params)
, BinaryCacheStoreConfig(params)
const StoreReference::Params & params)
: Store::Config{params}
, BinaryCacheStoreConfig{*this, params}
, cacheUri(
std::string { scheme }
+ "://"
@ -35,10 +54,9 @@ std::string HttpBinaryCacheStoreConfig::doc()
}
class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore
class HttpBinaryCacheStore :
public virtual BinaryCacheStore
{
private:
struct State
{
bool enabled = true;
@ -49,37 +67,41 @@ private:
public:
HttpBinaryCacheStore(
std::string_view scheme,
PathView cacheUri,
const Params & params)
: StoreConfig(params)
, BinaryCacheStoreConfig(params)
, HttpBinaryCacheStoreConfig(scheme, cacheUri, params)
, Store(params)
, BinaryCacheStore(params)
using Config = HttpBinaryCacheStoreConfig;
ref<const Config> config;
HttpBinaryCacheStore(ref<const Config> config)
: Store{*config}
, BinaryCacheStore{*config}
, config{config}
{
diskCache = getNarInfoDiskCache();
init();
}
std::string getUri() override
{
return cacheUri;
return config->cacheUri;
}
void init() override
{
// FIXME: do this lazily?
if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) {
wantMassQuery.setDefault(cacheInfo->wantMassQuery);
priority.setDefault(cacheInfo->priority);
if (auto cacheInfo = diskCache->upToDateCacheExists(config->cacheUri)) {
resolvedSubstConfig.wantMassQuery.value =
config->storeConfig.wantMassQuery.optValue.value_or(cacheInfo->wantMassQuery);
resolvedSubstConfig.priority.value =
config->storeConfig.priority.optValue.value_or(cacheInfo->priority);
} else {
try {
BinaryCacheStore::init();
} catch (UploadToHTTP &) {
throw Error("'%s' does not appear to be a binary cache", cacheUri);
throw Error("'%s' does not appear to be a binary cache", config->cacheUri);
}
diskCache->createCache(cacheUri, storeDir, wantMassQuery, priority);
diskCache->createCache(
config->cacheUri, storeDir, resolvedSubstConfig.wantMassQuery, resolvedSubstConfig.priority);
}
}
@ -137,7 +159,7 @@ protected:
try {
getFileTransfer()->upload(req);
} catch (FileTransferError & e) {
throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg());
throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", config->cacheUri, e.msg());
}
}
@ -146,7 +168,7 @@ protected:
return FileTransferRequest(
hasPrefix(path, "https://") || hasPrefix(path, "http://") || hasPrefix(path, "file://")
? path
: cacheUri + "/" + path);
: config->cacheUri + "/" + path);
}
@ -221,6 +243,11 @@ protected:
}
};
static RegisterStoreImplementation<HttpBinaryCacheStore, HttpBinaryCacheStoreConfig> regHttpBinaryCacheStore;
ref<Store> HttpBinaryCacheStore::Config::openStore() const
{
return make_ref<HttpBinaryCacheStore>(ref{shared_from_this()});
}
static RegisterStoreImplementation<HttpBinaryCacheStore::Config> regHttpBinaryCacheStore;
}

View File

@ -13,48 +13,39 @@ namespace nix {
struct NarInfo;
struct BinaryCacheStoreConfig : virtual StoreConfig
template<template<typename> class F>
struct BinaryCacheStoreConfigT
{
using StoreConfig::StoreConfig;
const Setting<std::string> compression{this, "xz", "compression",
"NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."};
const Setting<bool> writeNARListing{this, false, "write-nar-listing",
"Whether to write a JSON file that lists the files in each NAR."};
const Setting<bool> writeDebugInfo{this, false, "index-debug-info",
R"(
Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to
fetch debug info on demand
)"};
const Setting<Path> secretKeyFile{this, "", "secret-key",
"Path to the secret key used to sign the binary cache."};
const Setting<Path> localNarCache{this, "", "local-nar-cache",
"Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."};
const Setting<bool> parallelCompression{this, false, "parallel-compression",
"Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."};
const Setting<int> compressionLevel{this, -1, "compression-level",
R"(
The *preset level* to be used when compressing NARs.
The meaning and accepted values depend on the compression method selected.
`-1` specifies that the default compression level should be used.
)"};
F<std::string> compression;
F<bool> writeNARListing;
F<bool> writeDebugInfo;
F<Path> secretKeyFile;
F<Path> localNarCache;
F<bool> parallelCompression;
F<int> compressionLevel;
};
struct BinaryCacheStoreConfig :
BinaryCacheStoreConfigT<config::JustValue>
{
static config::SettingDescriptionMap descriptions();
const Store::Config & storeConfig;
BinaryCacheStoreConfig(const Store::Config &, const StoreReference::Params &);
};
/**
* @note subclasses must implement at least one of the two
* virtual getFile() methods.
*/
class BinaryCacheStore : public virtual BinaryCacheStoreConfig,
public virtual Store,
public virtual LogStore
struct BinaryCacheStore :
virtual Store,
virtual LogStore
{
using Config = BinaryCacheStoreConfig;
const Config & config;
private:
std::unique_ptr<Signer> signer;
@ -66,7 +57,7 @@ protected:
const std::string cacheInfoFile = "nix-cache-info";
BinaryCacheStore(const Params & params);
BinaryCacheStore(const Config &);
public:
@ -104,7 +95,11 @@ public:
public:
virtual void init() override;
/**
* Perform any necessary effectful operation to make the store up and
* running
*/
virtual void init();
private:

View File

@ -7,27 +7,27 @@ namespace nix {
class SSHMaster;
struct CommonSSHStoreConfig : virtual StoreConfig
template<template<typename> class F>
struct CommonSSHStoreConfigT
{
using StoreConfig::StoreConfig;
F<Path> sshKey;
F<std::string> sshPublicHostKey;
F<bool> compress;
F<std::string> remoteStore;
};
CommonSSHStoreConfig(std::string_view scheme, std::string_view host, const Params & params);
struct CommonSSHStoreConfig : CommonSSHStoreConfigT<config::JustValue>
{
static config::SettingDescriptionMap descriptions();
const Setting<Path> sshKey{this, "", "ssh-key",
"Path to the SSH private key used to authenticate to the remote machine."};
const Setting<std::string> sshPublicHostKey{this, "", "base64-ssh-public-host-key",
"The public host key of the remote machine."};
const Setting<bool> compress{this, false, "compress",
"Whether to enable SSH compression."};
const Setting<std::string> remoteStore{this, "", "remote-store",
R"(
[Store URL](@docroot@/store/types/index.md#store-url-format)
to be used on the remote machine. The default is `auto`
(i.e. use the Nix daemon or `/nix/store` directly).
)"};
/**
* @param scheme Note this isn't stored by this mix-in class, but
* just used for better error messages.
*/
CommonSSHStoreConfig(
std::string_view scheme,
std::string_view host,
const StoreReference::Params & params);
/**
* The `parseURL` function supports both IPv6 URIs as defined in
@ -56,7 +56,7 @@ struct CommonSSHStoreConfig : virtual StoreConfig
*/
SSHMaster createSSHMaster(
bool useMaster,
Descriptor logFD = INVALID_DESCRIPTOR);
Descriptor logFD = INVALID_DESCRIPTOR) const;
};
}

View File

@ -0,0 +1,69 @@
#pragma once
//@file
#include <nlohmann/json.hpp>
#include "nix/store/config-parse.hh"
#include "nix/util/util.hh"
#include "nix/util/configuration.hh"
namespace nix::config {
template<typename T>
OptValue<T>
SettingInfo<T>::parseConfig(const nlohmann::json::object_t & map, const ExperimentalFeatureSettings & xpSettings) const
{
const nlohmann::json * p = get(map, name);
if (p && experimentalFeature)
xpSettings.require(*experimentalFeature);
return {.optValue = p ? (std::optional<T>{p->get<T>()}) : std::nullopt};
}
template<typename T>
std::pair<std::string, SettingDescription> SettingInfo<T>::describe(const JustValue<T> & def) const
{
return {
std::string{name},
SettingDescription{
.description = stripIndentation(description),
.experimentalFeature = experimentalFeature,
.info =
SettingDescription::Single{
.defaultValue = documentDefault ? (std::optional{nlohmann::json(def.value)})
: (std::optional<nlohmann::json>{}),
},
},
};
}
/**
* Look up the setting's name in a map, falling back on the default if
* it does not exist.
*/
#define CONFIG_ROW(FIELD) .FIELD = descriptions.FIELD.parseConfig(params, xpSettings)
#define APPLY_ROW(FIELD) .FIELD = {.value = parsed.FIELD.optValue.value_or(std::move(defaults.FIELD))}
#define DESC_ROW(FIELD) \
{ \
descriptions.FIELD.describe(defaults.FIELD), \
}
#define MAKE_PARSE(CAPITAL, LOWER, FIELDS) \
static CAPITAL##T<config::OptValue> LOWER##Parse( \
const StoreReference::Params & params, \
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings) \
{ \
constexpr auto & descriptions = LOWER##Descriptions; \
return {FIELDS(CONFIG_ROW)}; \
}
#define MAKE_APPLY_PARSE(CAPITAL, LOWER, FIELDS) \
static CAPITAL##T<config::JustValue> LOWER##ApplyParse(const StoreReference::Params & params) \
{ \
auto defaults = LOWER##Defaults(); \
auto parsed = LOWER##Parse(params); \
return {FIELDS(APPLY_ROW)}; \
}
}

View File

@ -0,0 +1,144 @@
#pragma once
//@file
#include <nlohmann/json.hpp>
#include "nix/util/config-abstract.hh"
#include "nix/util/json-impls.hh"
#include "nix/util/experimental-features.hh"
namespace nix {
struct ExperimentalFeatureSettings;
};
namespace nix::config {
struct SettingDescription;
/**
* Typed version used as source of truth, and for operations like
* defaulting configurations.
*
* It is important that this type support `constexpr` values to avoid
* running into issues with static initialization order.
*/
template<typename T>
struct SettingInfo
{
/**
* Name of the setting, used when parsing configuration maps
*/
std::string_view name;
/**
* Description of the setting. It is used just for documentation.
*/
std::string_view description;
#if 0
/**
* Other names of the setting also used when parsing configuration
* maps. This is useful for back-compat, etc.
*/
std::set<std::string_view> aliases;
#endif
/**
* `ExperimentalFeature` that must be enabled if the setting is
* allowed to be used
*/
std::optional<ExperimentalFeature> experimentalFeature;
/**
* Whether to document the default value. (Some defaults are
* system-specific and should not be documented.)
*/
bool documentDefault = true;
/**
* Describe the setting as a key-value pair (name -> other info).
* The default value will be rendered to JSON if it is to be
* documented.
*/
std::pair<std::string, SettingDescription> describe(const JustValue<T> & def) const;
OptValue<T> parseConfig(const nlohmann::json::object_t & map, const ExperimentalFeatureSettings & xpSettings) const;
};
struct SettingDescription;
/**
* Map of setting names to descriptions of those settings.
*/
using SettingDescriptionMap = std::map<std::string, SettingDescription>;
/**
* Untyped version used for rendering docs. This is not the source of
* truth, it is generated from the typed one.
*
* @note No `name` field because this is intended to be used as the value type
* of a map
*/
struct SettingDescription
{
/**
* @see SettingInfo::description
*/
std::string description;
#if 0
/**
* @see SettingInfo::aliases
*/
std::set<std::string> aliases;
#endif
/**
* @see SettingInfo::experimentalFeature
*/
std::optional<ExperimentalFeature> experimentalFeature;
/**
* A single leaf setting, to be optionally specified by arbitrary
* value (of some type) or left default.
*/
struct Single
{
/**
* Optional, for the `SettingInfo::documentDefault = false` case.
*/
std::optional<nlohmann::json> defaultValue;
};
/**
* A nested settings object
*/
struct Sub
{
/**
* If `false`, this is just pure namespaceing. If `true`, we
* have a distinction between `null` and `{}`, meaning
* enabling/disabling the entire settings group.
*/
bool nullable = true;
SettingDescriptionMap map;
};
/**
* Variant for `info` below
*/
using Info = std::variant<Single, Sub>;
/**
* More information about this setting, depending on whether its the
* single leaf setting or subsettings case
*/
Info info;
};
}
JSON_IMPL(config::SettingDescription)

View File

@ -0,0 +1,24 @@
#include "nix/store/store-api.hh"
namespace nix {
struct DummyStoreConfig : std::enable_shared_from_this<DummyStoreConfig>, StoreConfig
{
DummyStoreConfig(std::string_view scheme, std::string_view authority, const StoreReference::Params & params);
static const std::string name()
{
return "Dummy Store";
}
static std::string doc();
static std::set<std::string> uriSchemes()
{
return {"dummy"};
}
ref<Store> openStore() const override;
};
}

View File

@ -2,29 +2,27 @@
namespace nix {
struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this<HttpBinaryCacheStoreConfig>,
Store::Config,
BinaryCacheStoreConfig
{
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
static config::SettingDescriptionMap descriptions();
HttpBinaryCacheStoreConfig(std::string_view scheme, std::string_view _cacheUri, const Params & params);
HttpBinaryCacheStoreConfig(
std::string_view scheme, std::string_view cacheUri, const StoreReference::Params & params);
Path cacheUri;
const std::string name() override
static const std::string name()
{
return "HTTP Binary Cache Store";
}
static std::set<std::string> uriSchemes()
{
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
auto ret = std::set<std::string>({"http", "https"});
if (forceHttp)
ret.insert("file");
return ret;
}
static std::set<std::string> uriSchemes();
std::string doc() override;
static std::string doc();
ref<Store> openStore() const override;
};
}

View File

@ -10,20 +10,31 @@
namespace nix {
struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
template<template<typename> class F>
struct LegacySSHStoreConfigT
{
using CommonSSHStoreConfig::CommonSSHStoreConfig;
F<Strings> remoteProgram;
F<int> maxConnections;
};
struct LegacySSHStoreConfig :
std::enable_shared_from_this<LegacySSHStoreConfig>,
Store::Config,
CommonSSHStoreConfig,
LegacySSHStoreConfigT<config::JustValue>
{
static config::SettingDescriptionMap descriptions();
/**
* Hack for getting remote build log output. Intentionally not a
* documented user-visible setting.
*/
Descriptor logFD = INVALID_DESCRIPTOR;
LegacySSHStoreConfig(
std::string_view scheme,
std::string_view authority,
const Params & params);
const Setting<Strings> remoteProgram{this, {"nix-store"}, "remote-program",
"Path to the `nix-store` executable on the remote machine."};
const Setting<int> maxConnections{this, 1, "max-connections",
"Maximum number of concurrent SSH connections."};
const StoreReference::Params & params);
/**
* Hack for hydra
@ -35,23 +46,20 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
*/
std::optional<size_t> connPipeSize;
const std::string name() override { return "SSH Store"; }
static const std::string name() { return "SSH Store"; }
static std::set<std::string> uriSchemes() { return {"ssh"}; }
std::string doc() override;
static std::string doc();
ref<Store> openStore() const override;
};
struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store
struct LegacySSHStore : public virtual Store
{
#ifndef _WIN32
// Hack for getting remote build log output.
// Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
// the documentation
const Setting<int> logFD{this, INVALID_DESCRIPTOR, "log-fd", "file descriptor to which SSH's stderr is connected"};
#else
Descriptor logFD = INVALID_DESCRIPTOR;
#endif
using Config = LegacySSHStoreConfig;
ref<const Config> config;
struct Connection;
@ -59,10 +67,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
SSHMaster master;
LegacySSHStore(
std::string_view scheme,
std::string_view host,
const Params & params);
LegacySSHStore(ref<const Config>);
ref<Connection> openConnection();
@ -187,10 +192,7 @@ public:
* The legacy ssh protocol doesn't support checking for trusted-user.
* Try using ssh-ng:// instead if you want to know.
*/
std::optional<TrustedFlag> isTrustedClient() override
{
return std::nullopt;
}
std::optional<TrustedFlag> isTrustedClient() override;
void queryRealisationUncached(const DrvOutput &,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override

View File

@ -2,22 +2,31 @@
namespace nix {
struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
struct LocalBinaryCacheStoreConfig : std::enable_shared_from_this<LocalBinaryCacheStoreConfig>,
Store::Config,
BinaryCacheStoreConfig
{
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
static config::SettingDescriptionMap descriptions();
LocalBinaryCacheStoreConfig(std::string_view scheme, PathView binaryCacheDir, const Params & params);
/**
* @param binaryCacheDir `file://` is a short-hand for `file:///`
* for now.
*/
LocalBinaryCacheStoreConfig(
std::string_view scheme, PathView binaryCacheDir, const StoreReference::Params & params);
Path binaryCacheDir;
const std::string name() override
static const std::string name()
{
return "Local Binary Cache Store";
}
static std::set<std::string> uriSchemes();
std::string doc() override;
static std::string doc();
ref<Store> openStore() const override;
};
}

View File

@ -7,9 +7,24 @@
namespace nix {
struct LocalFSStoreConfig : virtual StoreConfig
template<template<typename> class F>
struct LocalFSStoreConfigT
{
using StoreConfig::StoreConfig;
F<std::optional<Path>> rootDir;
F<Path> stateDir;
F<Path> logDir;
F<Path> realStoreDir;
};
struct LocalFSStoreConfig : LocalFSStoreConfigT<config::JustValue>
{
const Store::Config & storeConfig;
static config::SettingDescriptionMap descriptions();
LocalFSStoreConfig(
const Store::Config & storeConfig,
const StoreReference::Params &);
/**
* Used to override the `root` settings. Can't be done via modifying
@ -18,38 +33,26 @@ struct LocalFSStoreConfig : virtual StoreConfig
*
* @todo Make this less error-prone with new store settings system.
*/
LocalFSStoreConfig(PathView path, const Params & params);
const OptionalPathSetting rootDir{this, std::nullopt,
"root",
"Directory prefixed to all other paths."};
const PathSetting stateDir{this,
rootDir.get() ? *rootDir.get() + "/nix/var/nix" : settings.nixStateDir,
"state",
"Directory where Nix will store state."};
const PathSetting logDir{this,
rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : settings.nixLogDir,
"log",
"directory where Nix will store log files."};
const PathSetting realStoreDir{this,
rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real",
"Physical path of the Nix store."};
LocalFSStoreConfig(
const Store::Config & storeConfig,
PathView path,
const StoreReference::Params & params);
};
class LocalFSStore : public virtual LocalFSStoreConfig,
public virtual Store,
public virtual GcStore,
public virtual LogStore
struct LocalFSStore :
virtual Store,
virtual GcStore,
virtual LogStore
{
public:
using Config = LocalFSStoreConfig;
const Config & config;
inline static std::string operationName = "Local Filesystem Store";
const static std::string drvsLogDir;
LocalFSStore(const Params & params);
LocalFSStore(const Config & params);
void narFromPath(const StorePath & path, Sink & sink) override;
ref<SourceAccessor> getFSAccessor(bool requireValidPath = true) override;
@ -70,7 +73,7 @@ public:
*/
virtual Path addPermRoot(const StorePath & storePath, const Path & gcRoot) = 0;
virtual Path getRealStoreDir() { return realStoreDir; }
virtual Path getRealStoreDir() { return config.realStoreDir; }
Path toRealPath(const Path & storePath) override
{

View File

@ -2,63 +2,33 @@
namespace nix {
template<template<typename> class F>
struct LocalOverlayStoreConfigT
{
const F<ref<const StoreConfig>> lowerStoreConfig;
const F<Path> upperLayer;
const F<bool> checkMount;
const F<Path> remountHook;
};
/**
* Configuration for `LocalOverlayStore`.
*/
struct LocalOverlayStoreConfig : virtual LocalStoreConfig
struct LocalOverlayStoreConfig :
LocalStoreConfig,
LocalOverlayStoreConfigT<config::JustValue>
{
LocalOverlayStoreConfig(const StringMap & params)
: LocalOverlayStoreConfig("local-overlay", "", params)
{ }
static config::SettingDescriptionMap descriptions();
LocalOverlayStoreConfig(std::string_view scheme, PathView path, const Params & params)
: StoreConfig(params)
, LocalFSStoreConfig(path, params)
, LocalStoreConfig(scheme, path, params)
{
}
LocalOverlayStoreConfig(
std::string_view scheme,
PathView path,
const StoreReference::Params & params);
const Setting<std::string> lowerStoreUri{(StoreConfig*) this, "", "lower-store",
R"(
[Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly).
static const std::string name() { return "Experimental Local Overlay Store"; }
Must be a store with a store dir on the file system.
Must be used as OverlayFS lower layer for this store's store dir.
)"};
const PathSetting upperLayer{(StoreConfig*) this, "", "upper-layer",
R"(
Directory containing the OverlayFS upper layer for this store's store dir.
)"};
Setting<bool> checkMount{(StoreConfig*) this, true, "check-mount",
R"(
Check that the overlay filesystem is correctly mounted.
Nix does not manage the overlayfs mount point itself, but the correct
functioning of the overlay store does depend on this mount point being set up
correctly. Rather than just assume this is the case, check that the lowerdir
and upperdir options are what we expect them to be. This check is on by
default, but can be disabled if needed.
)"};
const PathSetting remountHook{(StoreConfig*) this, "", "remount-hook",
R"(
Script or other executable to run when overlay filesystem needs remounting.
This is occasionally necessary when deleting a store path that exists in both upper and lower layers.
In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly
is the only way to perform the deletion without creating a "whiteout".
However this causes the OverlayFS kernel data structures to get out-of-sync,
and can lead to 'stale file handle' errors; remounting solves the problem.
The store directory is passed as an argument to the invoked executable.
)"};
const std::string name() override { return "Experimental Local Overlay Store"; }
std::optional<ExperimentalFeature> experimentalFeature() const override
static std::optional<ExperimentalFeature> experimentalFeature()
{
return ExperimentalFeature::LocalOverlayStore;
}
@ -68,7 +38,9 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig
return { "local-overlay" };
}
std::string doc() override;
static std::string doc();
ref<Store> openStore() const override;
protected:
/**
@ -79,7 +51,9 @@ protected:
* at that file path. It might be stored in the lower layer instead,
* or it might not be part of this store at all.
*/
Path toUpperPath(const StorePath & path);
Path toUpperPath(const StorePath & path) const;
friend struct LocalOverlayStore;
};
/**
@ -88,8 +62,20 @@ protected:
* Documentation on overridden methods states how they differ from their
* `LocalStore` counterparts.
*/
class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual LocalStore
struct LocalOverlayStore : virtual LocalStore
{
using Config = LocalOverlayStoreConfig;
ref<const Config> config;
LocalOverlayStore(ref<const Config>);
std::string getUri() override
{
return "local-overlay://";
}
private:
/**
* The store beneath us.
*
@ -99,20 +85,6 @@ class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual
*/
ref<LocalFSStore> lowerStore;
public:
LocalOverlayStore(const Params & params)
: LocalOverlayStore("local-overlay", "", params)
{
}
LocalOverlayStore(std::string_view scheme, PathView path, const Params & params);
std::string getUri() override
{
return "local-overlay://";
}
private:
/**
* First copy up any lower store realisation with the same key, so we
* merge rather than mask it.

View File

@ -34,49 +34,55 @@ struct OptimiseStats
uint64_t bytesFreed = 0;
};
struct LocalStoreConfig : virtual LocalFSStoreConfig
template<template<typename> class F>
struct LocalStoreConfigT
{
using LocalFSStoreConfig::LocalFSStoreConfig;
F<bool> requireSigs;
F<bool> readOnly;
};
struct LocalStoreConfig :
std::enable_shared_from_this<LocalStoreConfig>,
Store::Config,
LocalFSStore::Config,
LocalStoreConfigT<config::JustValue>
{
static config::SettingDescriptionMap descriptions();
LocalStoreConfig(const StoreReference::Params & params)
: LocalStoreConfig{"local", "", params}
{}
LocalStoreConfig(
std::string_view scheme,
std::string_view authority,
const Params & params);
const StoreReference::Params & params);
Setting<bool> requireSigs{this,
settings.requireSigs,
"require-sigs",
"Whether store paths copied into this store should have a trusted signature."};
/**
* For `RestrictedStore`
*/
LocalStoreConfig(const LocalStoreConfig &);
Setting<bool> readOnly{this,
false,
"read-only",
R"(
Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem.
Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem.
Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set.
> **Warning**
> Do not use this unless the filesystem is read-only.
>
> Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process.
> While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it.
)"};
const std::string name() override { return "Local Store"; }
static const std::string name() { return "Local Store"; }
static std::set<std::string> uriSchemes()
{ return {"local"}; }
std::string doc() override;
static std::string doc();
ref<Store> openStore() const override;
};
class LocalStore : public virtual LocalStoreConfig
, public virtual IndirectRootStore
, public virtual GcStore
class LocalStore :
public virtual IndirectRootStore,
public virtual GcStore
{
public:
using Config = LocalStoreConfig;
ref<const LocalStoreConfig> config;
private:
/**
@ -144,11 +150,7 @@ public:
* Initialise the local store, upgrading the schema if
* necessary.
*/
LocalStore(const Params & params);
LocalStore(
std::string_view scheme,
PathView path,
const Params & params);
LocalStore(ref<const Config> params);
~LocalStore();

View File

@ -1,6 +1,8 @@
#pragma once
///@file
#include <nlohmann/json.hpp>
#include "nix/util/ref.hh"
#include "nix/store/store-reference.hh"

View File

@ -22,13 +22,16 @@ headers = [config_pub_h] + files(
'common-protocol-impl.hh',
'common-protocol.hh',
'common-ssh-store-config.hh',
'config-parse-impl.hh',
'config-parse.hh',
'content-address.hh',
'daemon.hh',
'derivations.hh',
'derivation-options.hh',
'derivations.hh',
'derived-path-map.hh',
'derived-path.hh',
'downstream-placeholder.hh',
'dummy-store.hh',
'filetransfer.hh',
'gc-store.hh',
'globals.hh',
@ -65,16 +68,18 @@ headers = [config_pub_h] + files(
'restricted-store.hh',
's3-binary-cache-store.hh',
's3.hh',
'ssh-store.hh',
'serve-protocol-connection.hh',
'serve-protocol-impl.hh',
'serve-protocol.hh',
'sqlite.hh',
'ssh-store.hh',
'ssh.hh',
'store-api.hh',
'store-cast.hh',
'store-dir-config.hh',
'store-open.hh',
'store-reference.hh',
'store-registration.hh',
'uds-remote-store.hh',
'worker-protocol-connection.hh',
'worker-protocol-impl.hh',

View File

@ -86,7 +86,7 @@ typedef std::list<Generation> Generations;
*/
std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile);
class LocalFSStore;
struct LocalFSStore;
/**
* Create a new generation of the given profile

View File

@ -19,7 +19,7 @@ class RemoteFSAccessor : public SourceAccessor
std::pair<ref<SourceAccessor>, CanonPath> fetch(const CanonPath & path);
friend class BinaryCacheStore;
friend struct BinaryCacheStore;
Path makeCacheFile(std::string_view hashPart, const std::string & ext);

View File

@ -18,31 +18,36 @@ struct FdSink;
struct FdSource;
template<typename T> class Pool;
struct RemoteStoreConfig : virtual StoreConfig
template<template<typename> class F>
struct RemoteStoreConfigT
{
using StoreConfig::StoreConfig;
F<int> maxConnections;
F<unsigned int> maxConnectionAge;
};
const Setting<int> maxConnections{this, 1, "max-connections",
"Maximum number of concurrent connections to the Nix daemon."};
struct RemoteStoreConfig : RemoteStoreConfigT<config::JustValue>
{
static config::SettingDescriptionMap descriptions();
const Setting<unsigned int> maxConnectionAge{this,
std::numeric_limits<unsigned int>::max(),
"max-connection-age",
"Maximum age of a connection before it is closed."};
const Store::Config & storeConfig;
RemoteStoreConfig(const Store::Config &, const StoreReference::Params &);
};
/**
* \todo RemoteStore is a misnomer - should be something like
* DaemonStore.
*/
class RemoteStore : public virtual RemoteStoreConfig,
struct RemoteStore :
public virtual Store,
public virtual GcStore,
public virtual LogStore
{
public:
using Config = RemoteStoreConfig;
RemoteStore(const Params & params);
const Config & config;
RemoteStore(const Config & config);
/* Implementations of abstract store API methods. */

View File

@ -1,7 +1,7 @@
#pragma once
///@file
#include "local-store.hh"
#include "nix/store/local-store.hh"
namespace nix {
@ -55,6 +55,6 @@ struct RestrictionContext
/**
* Create a shared pointer to a restricted store.
*/
ref<Store> makeRestrictedStore(const Store::Params & params, ref<LocalStore> next, RestrictionContext & context);
ref<Store> makeRestrictedStore(ref<LocalStore::Config> config, ref<LocalStore> next, RestrictionContext & context);
}

View File

@ -11,89 +11,33 @@
namespace nix {
struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
template<template<typename> class F>
struct S3BinaryCacheStoreConfigT
{
F<std::string> profile;
F<std::string> region;
F<std::string> scheme;
F<std::string> endpoint;
F<std::string> narinfoCompression;
F<std::string> lsCompression;
F<std::string> logCompression;
F<bool> multipartUpload;
F<uint64_t> bufferSize;
};
struct S3BinaryCacheStoreConfig : std::enable_shared_from_this<S3BinaryCacheStoreConfig>,
Store::Config,
BinaryCacheStoreConfig,
S3BinaryCacheStoreConfigT<config::JustValue>
{
static config::SettingDescriptionMap descriptions();
S3BinaryCacheStoreConfig(
std::string_view uriScheme, std::string_view bucketName, const StoreReference::Params & params);
std::string bucketName;
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
S3BinaryCacheStoreConfig(std::string_view uriScheme, std::string_view bucketName, const Params & params);
const Setting<std::string> profile{
this,
"",
"profile",
R"(
The name of the AWS configuration profile to use. By default
Nix will use the `default` profile.
)"};
protected:
constexpr static const char * defaultRegion = "us-east-1";
public:
const Setting<std::string> region{
this,
defaultRegion,
"region",
R"(
The region of the S3 bucket. If your bucket is not in
`useast-1`, you should always explicitly specify the region
parameter.
)"};
const Setting<std::string> scheme{
this,
"",
"scheme",
R"(
The scheme used for S3 requests, `https` (default) or `http`. This
option allows you to disable HTTPS for binary caches which don't
support it.
> **Note**
>
> HTTPS should be used if the cache might contain sensitive
> information.
)"};
const Setting<std::string> endpoint{
this,
"",
"endpoint",
R"(
The URL of the endpoint of an S3-compatible service such as MinIO.
Do not specify this setting if you're using Amazon S3.
> **Note**
>
> This endpoint must support HTTPS and will use path-based
> addressing instead of virtual host based addressing.
)"};
const Setting<std::string> narinfoCompression{
this, "", "narinfo-compression", "Compression method for `.narinfo` files."};
const Setting<std::string> lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."};
const Setting<std::string> logCompression{
this,
"",
"log-compression",
R"(
Compression method for `log/*` files. It is recommended to
use a compression method supported by most web browsers
(e.g. `brotli`).
)"};
const Setting<bool> multipartUpload{this, false, "multipart-upload", "Whether to use multi-part uploads."};
const Setting<uint64_t> bufferSize{
this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."};
const std::string name() override
static std::string name()
{
return "S3 Binary Cache Store";
}
@ -103,16 +47,18 @@ public:
return {"s3"};
}
std::string doc() override;
static std::string doc();
ref<Store> openStore() const override;
};
class S3BinaryCacheStore : public virtual BinaryCacheStore
struct S3BinaryCacheStore : virtual BinaryCacheStore
{
protected:
using Config = S3BinaryCacheStoreConfig;
S3BinaryCacheStore(const Params & params);
ref<const Config> config;
public:
S3BinaryCacheStore(ref<const Config>);
struct Stats
{

View File

@ -8,17 +8,29 @@
namespace nix {
struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig
template<template<typename> class F>
struct SSHStoreConfigT
{
using CommonSSHStoreConfig::CommonSSHStoreConfig;
using RemoteStoreConfig::RemoteStoreConfig;
F<Strings> remoteProgram;
};
SSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params);
struct SSHStoreConfig : std::enable_shared_from_this<SSHStoreConfig>,
Store::Config,
RemoteStore::Config,
CommonSSHStoreConfig,
SSHStoreConfigT<config::JustValue>
{
static config::SettingDescriptionMap descriptions();
const Setting<Strings> remoteProgram{
this, {"nix-daemon"}, "remote-program", "Path to the `nix-daemon` executable on the remote machine."};
std::optional<LocalFSStore::Config> mounted;
const std::string name() override
SSHStoreConfig(
std::string_view scheme,
std::string_view authority,
const StoreReference::Params & params,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
static const std::string name()
{
return "Experimental SSH Store";
}
@ -28,34 +40,9 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig
return {"ssh-ng"};
}
std::string doc() override;
};
static std::string doc();
struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig
{
using LocalFSStoreConfig::LocalFSStoreConfig;
using SSHStoreConfig::SSHStoreConfig;
MountedSSHStoreConfig(StringMap params);
MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params);
const std::string name() override
{
return "Experimental SSH Store with filesystem mounted";
}
static std::set<std::string> uriSchemes()
{
return {"mounted-ssh-ng"};
}
std::string doc() override;
std::optional<ExperimentalFeature> experimentalFeature() const override
{
return ExperimentalFeature::MountedSSHStore;
}
ref<Store> openStore() const override;
};
}

View File

@ -9,7 +9,6 @@
#include "nix/util/lru-cache.hh"
#include "nix/util/sync.hh"
#include "nix/store/globals.hh"
#include "nix/util/configuration.hh"
#include "nix/store/path-info.hh"
#include "nix/util/repair-flag.hh"
#include "nix/store/store-dir-config.hh"
@ -41,7 +40,7 @@ namespace nix {
* 2. A class `Foo : virtual Store, virtual FooConfig` that contains the
* implementation of the store.
*
* This class is expected to have a constructor `Foo(const Params & params)`
* This class is expected to have a constructor `Foo(const StoreReference::Params & params)`
* that calls `StoreConfig(params)` (otherwise you're gonna encounter an
* `assertion failure` when trying to instantiate it).
*
@ -97,27 +96,52 @@ struct KeyedBuildResult;
typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
struct StoreConfig : public StoreDirConfig
template<template<typename> class F>
struct StoreConfigT
{
using Params = StoreReference::Params;
F<int> pathInfoCacheSize;
F<bool> isTrusted;
F<StringSet> systemFeatures;
};
using StoreDirConfig::StoreDirConfig;
template<template<typename> class F>
struct SubstituterConfigT
{
F<int> priority;
F<bool> wantMassQuery;
};
StoreConfig() = delete;
/**
* @note In other cases we don't expose this function directly, but in
* this case we must because of `Store::resolvedSubstConfig` below. As
* the docs of that field describe, this is a case where the
* configuration is intentionally stateful.
*/
SubstituterConfigT<config::JustValue> substituterConfigDefaults();
static StringSet getDefaultSystemFeatures();
/**
* @note `config::OptValue` rather than `config::JustValue` is applied to
* `SubstitutorConfigT` because these are overrides. Caches themselves (not our
* config) can update default settings, but aren't allowed to update settings
* specified by the client (i.e. us).
*/
struct StoreConfig :
StoreDirConfig,
StoreConfigT<config::JustValue>,
SubstituterConfigT<config::OptValue>
{
static config::SettingDescriptionMap descriptions();
StoreConfig(const StoreReference::Params &);
virtual ~StoreConfig() { }
/**
* The name of this type of store.
*/
virtual const std::string name() = 0;
static StringSet getDefaultSystemFeatures();
/**
* Documentation for this type of store.
*/
virtual std::string doc()
static std::string doc()
{
return "";
}
@ -126,47 +150,50 @@ struct StoreConfig : public StoreDirConfig
* An experimental feature this type store is gated, if it is to be
* experimental.
*/
virtual std::optional<ExperimentalFeature> experimentalFeature() const
static std::optional<ExperimentalFeature> experimentalFeature()
{
return std::nullopt;
}
const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size",
"Size of the in-memory store path metadata cache."};
const Setting<bool> isTrusted{this, false, "trusted",
R"(
Whether paths from this store can be used as substitutes
even if they are not signed by a key listed in the
[`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys)
setting.
)"};
Setting<int> priority{this, 0, "priority",
R"(
Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters).
A lower value means a higher priority.
)"};
Setting<bool> wantMassQuery{this, false, "want-mass-query",
R"(
Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters).
)"};
Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(),
"system-features",
R"(
Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations.
Example: `"kvm"`
)",
{},
// Don't document the machine-specific default value
false};
/**
* Open a store of the type corresponding to this configuration
* type.
*/
virtual ref<Store> openStore() const = 0;
};
class Store : public std::enable_shared_from_this<Store>, public virtual StoreConfig
/**
* A Store (client)
*
* This is an interface type allowing for create and read operations on
* a collection of store objects, and also building new store objects
* from `Derivation`s. See the manual for further details.
*
* "client" used is because this is just one view/actor onto an
* underlying resource, which could be an external process (daemon
* server), file system state, etc.
*/
class Store : public std::enable_shared_from_this<Store>, public MixStoreDirMethods
{
public:
using Config = StoreConfig;
const Config & config;
/**
* @note Avoid churn, since we used to inherit from `Config`.
*/
operator const Config &() const { return config; }
/**
* Resolved substituter configuration. This is intentionally mutable
* as store clients may do IO to ask the underlying store for their
* default setting values if the client config did not statically
* override them.
*/
SubstituterConfigT<config::JustValue> resolvedSubstConfig = substituterConfigDefaults();
protected:
struct PathInfoCacheValue {
@ -205,14 +232,9 @@ protected:
std::shared_ptr<NarInfoDiskCache> diskCache;
Store(const Params & params);
Store(const Store::Config & config);
public:
/**
* Perform any necessary effectful operation to make the store up and
* running
*/
virtual void init() {};
virtual ~Store() { }
@ -865,74 +887,6 @@ StorePath resolveDerivedPath(Store &, const SingleDerivedPath &, Store * evalSto
OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr);
/**
* @return a Store object to access the Nix store denoted by
* uri (slight misnomer...).
*/
ref<Store> openStore(StoreReference && storeURI);
/**
* Opens the store at `uri`, where `uri` is in the format expected by `StoreReference::parse`
*/
ref<Store> openStore(const std::string & uri = settings.storeUri.get(),
const Store::Params & extraParams = Store::Params());
/**
* @return the default substituter stores, defined by the
* substituters option and various legacy options.
*/
std::list<ref<Store>> getDefaultSubstituters();
struct StoreFactory
{
std::set<std::string> uriSchemes;
/**
* The `authorityPath` parameter is `<authority>/<path>`, or really
* whatever comes after `<scheme>://` and before `?<query-params>`.
*/
std::function<std::shared_ptr<Store> (
std::string_view scheme,
std::string_view authorityPath,
const Store::Params & params)> create;
std::function<std::shared_ptr<StoreConfig> ()> getConfig;
};
struct Implementations
{
static std::vector<StoreFactory> * registered;
template<typename T, typename TConfig>
static void add()
{
if (!registered) registered = new std::vector<StoreFactory>();
StoreFactory factory{
.uriSchemes = TConfig::uriSchemes(),
.create =
([](auto scheme, auto uri, auto & params)
-> std::shared_ptr<Store>
{ return std::make_shared<T>(scheme, uri, params); }),
.getConfig =
([]()
-> std::shared_ptr<StoreConfig>
{ return std::make_shared<TConfig>(StringMap({})); })
};
registered->push_back(factory);
}
};
template<typename T, typename TConfig>
struct RegisterStoreImplementation
{
RegisterStoreImplementation()
{
Implementations::add<T, TConfig>();
}
};
/**
* Display a set of paths in human-readable form (i.e., between quotes
* and separated by commas).
@ -954,3 +908,6 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
Store * evalStore = nullptr);
}
// Parses a Store URL, uses global state not pure so think about this
JSON_IMPL(ref<const StoreConfig>)

View File

@ -3,8 +3,8 @@
#include "nix/store/path.hh"
#include "nix/util/hash.hh"
#include "nix/store/content-address.hh"
#include "nix/store/globals.hh"
#include "nix/util/configuration.hh"
#include "nix/store/store-reference.hh"
#include "nix/store/config-parse.hh"
#include <map>
#include <string>
@ -18,24 +18,32 @@ struct SourcePath;
MakeError(BadStorePath, Error);
MakeError(BadStorePathName, BadStorePath);
struct StoreDirConfig : public Config
/**
* Underlying store directory configuration type.
*
* Don't worry to much about the `F` parameter, it just some abstract
* nonsense for the "higher-kinded data" pattern. It is used in each
* settings record in order to ensure don't forgot to parse or document
* settings field.
*/
template<template<typename> class F>
struct StoreDirConfigT
{
using Config::Config;
F<Path> _storeDir;
};
StoreDirConfig() = delete;
virtual ~StoreDirConfig() = default;
const PathSetting storeDir_{this, settings.nixStore,
"store",
R"(
Logical location of the Nix store, usually
`/nix/store`. Note that you can only copy store paths
between stores if they have the same `store` setting.
)"};
const Path storeDir = storeDir_;
// pure methods
/**
* @todo This should just be part of `StoreDirConfig`. However, it would
* be a huge amount of churn if `Store` didn't have these methods
* anymore, forcing a bunch of code to go from `store.method(...)` to
* `store.config.method(...)`.
*
* So we instead pull out the methods into their own mix-in, so can put
* them directly on the Store too.
*/
struct MixStoreDirMethods
{
const Path & storeDir;
StorePath parseStorePath(std::string_view path) const;
@ -56,7 +64,7 @@ struct StoreDirConfig : public Config
* Display a set of paths in human-readable form (i.e., between quotes
* and separated by commas).
*/
std::string showPaths(const StorePathSet & paths);
std::string showPaths(const StorePathSet & paths) const;
/**
* @return true if *path* is in the Nix store (but not the Nix
@ -104,4 +112,19 @@ struct StoreDirConfig : public Config
PathFilter & filter = defaultPathFilter) const;
};
/**
* Store directory configuration type.
*
* Combines the underlying `*T` type (with plain values for the fields)
* and the methods.
*/
struct StoreDirConfig : StoreDirConfigT<config::JustValue>, MixStoreDirMethods
{
static config::SettingDescriptionMap descriptions();
StoreDirConfig(const StoreReference::Params & params);
virtual ~StoreDirConfig() = default;
};
}

View File

@ -0,0 +1,43 @@
#pragma once
/**
* @file
*
* For opening a store described by an `StoreReference`, which is an "untyped"
* notion which needs to be decoded against a collection of specific
* implementations.
*
* For consumers of the store registration machinery defined in
* `store-registration.hh`. Not needed by store implementation definitions, or
* usages of a given `Store` which will be passed in.
*/
#include "nix/store/store-api.hh"
namespace nix {
/**
* @return The store config denoted `uri` (slight misnomer...).
*/
ref<StoreConfig> resolveStoreConfig(StoreReference && storeURI);
/**
* @return a Store object to access the Nix store denoted by
* uri (slight misnomer...).
*/
ref<Store> openStore(StoreReference && storeURI);
/**
* Opens the store at `uri`, where `uri` is in the format expected by
* `StoreReference::parse`
*/
ref<Store> openStore(
const std::string & uri = settings.storeUri.get(),
const StoreReference::Params & extraParams = StoreReference::Params());
/**
* @return the default substituter stores, defined by the
* substituters option and various legacy options.
*/
std::list<ref<Store>> getDefaultSubstituters();
}

View File

@ -2,8 +2,10 @@
///@file
#include <variant>
#include <nlohmann/json_fwd.hpp>
#include "nix/util/types.hh"
#include "nix/util/json-impls.hh"
namespace nix {
@ -41,7 +43,17 @@ namespace nix {
*/
struct StoreReference
{
using Params = std::map<std::string, std::string>;
/**
* Would do
*
* ```
* using Params = nlohmann::json::object_t;
* ```
*
* but cannot because `<nlohmann/json_fwd.hpp>` doesn't have that.
*
*/
using Params = std::map<std::string, nlohmann::json, std::less<>>;
/**
* Special store reference `""` or `"auto"`
@ -70,7 +82,7 @@ struct StoreReference
Params params;
bool operator==(const StoreReference & rhs) const = default;
bool operator==(const StoreReference & rhs) const;
/**
* Render the whole store reference as a URI, including parameters.
@ -89,3 +101,5 @@ struct StoreReference
std::pair<std::string, StoreReference::Params> splitUriAndParams(const std::string & uri);
}
JSON_IMPL(StoreReference)

View File

@ -0,0 +1,94 @@
#pragma once
/**
* @file
*
* Infrastructure for "registering" store implementations. Used by the
* store implementation definitions themselves but not by consumers of
* those implementations.
*/
#include "nix/store/store-api.hh"
namespace nix {
struct StoreFactory
{
/**
* Documentation for this type of store.
*/
std::string doc;
/**
* URIs with these schemes should be handled by this factory
*/
std::set<std::string> uriSchemes;
/**
* @note This is a functional pointer for now because this situation:
*
* - We register store types with global initializers
*
* - The default values for some settings maybe depend on the settings globals.
*
* And because the ordering of global initialization is arbitrary,
* this is not allowed. For now, we can simply defer actually
* creating these maps until we need to later.
*/
config::SettingDescriptionMap (*configDescriptions)();
/**
* An experimental feature this type store is gated, if it is to be
* experimental.
*/
std::optional<ExperimentalFeature> experimentalFeature;
/**
* The `authorityPath` parameter is `<authority>/<path>`, or really
* whatever comes after `<scheme>://` and before `?<query-params>`.
*/
std::function<ref<StoreConfig>(
std::string_view scheme, std::string_view authorityPath, const StoreReference::Params & params)>
parseConfig;
};
struct Implementations
{
private:
/**
* The name of this type of store, and a factory for it.
*/
using V = std::vector<std::pair<std::string, StoreFactory>>;
public:
static V * registered;
template<typename TConfig>
static void add()
{
if (!registered)
registered = new V{};
StoreFactory factory{
.doc = TConfig::doc(),
.uriSchemes = TConfig::uriSchemes(),
.configDescriptions = TConfig::descriptions,
.experimentalFeature = TConfig::experimentalFeature(),
.parseConfig = ([](auto scheme, auto uri, auto & params) -> ref<StoreConfig> {
return make_ref<TConfig>(scheme, uri, params);
}),
};
registered->push_back({TConfig::name(), std::move(factory)});
}
};
template<typename TConfig>
struct RegisterStoreImplementation
{
RegisterStoreImplementation()
{
Implementations::add<TConfig>();
}
};
}

View File

@ -7,12 +7,17 @@
namespace nix {
struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig
struct UDSRemoteStoreConfig :
std::enable_shared_from_this<UDSRemoteStoreConfig>,
Store::Config,
LocalFSStore::Config,
RemoteStore::Config
{
// TODO(fzakaria): Delete this constructor once moved over to the factory pattern
// outlined in https://github.com/NixOS/nix/issues/10766
using LocalFSStoreConfig::LocalFSStoreConfig;
using RemoteStoreConfig::RemoteStoreConfig;
static config::SettingDescriptionMap descriptions();
UDSRemoteStoreConfig(const StoreReference::Params & params)
: UDSRemoteStoreConfig{"unix", "", params}
{}
/**
* @param authority is the socket path.
@ -20,11 +25,11 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon
UDSRemoteStoreConfig(
std::string_view scheme,
std::string_view authority,
const Params & params);
const StoreReference::Params & params);
const std::string name() override { return "Local Daemon Store"; }
static const std::string name() { return "Local Daemon Store"; }
std::string doc() override;
static std::string doc();
/**
* The path to the unix domain socket.
@ -34,32 +39,21 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon
*/
Path path;
protected:
static constexpr char const * scheme = "unix";
public:
static std::set<std::string> uriSchemes()
{ return {scheme}; }
{ return {"unix"}; }
ref<Store> openStore() const override;
};
class UDSRemoteStore : public virtual UDSRemoteStoreConfig
, public virtual IndirectRootStore
, public virtual RemoteStore
struct UDSRemoteStore :
virtual IndirectRootStore,
virtual RemoteStore
{
public:
using Config = UDSRemoteStoreConfig;
/**
* @deprecated This is the old API to construct the store.
*/
UDSRemoteStore(const Params & params);
ref<const Config> config;
/**
* @param authority is the socket path.
*/
UDSRemoteStore(
std::string_view scheme,
std::string_view authority,
const Params & params);
UDSRemoteStore(ref<const Config>);
std::string getUri() override;

View File

@ -12,18 +12,76 @@
#include "nix/store/ssh.hh"
#include "nix/store/derivations.hh"
#include "nix/util/callback.hh"
#include "nix/store/config-parse-impl.hh"
#include "nix/store/store-registration.hh"
namespace nix {
LegacySSHStoreConfig::LegacySSHStoreConfig(
constexpr static const LegacySSHStoreConfigT<config::SettingInfo> legacySSHStoreConfigDescriptions = {
.remoteProgram{
.name = "remote-program",
.description = "Path to the `nix-store` executable on the remote machine.",
},
.maxConnections{
.name = "max-connections",
.description = "Maximum number of concurrent SSH connections.",
},
};
#define LEGACY_SSH_STORE_CONFIG_FIELDS(X) \
X(remoteProgram), \
X(maxConnections)
MAKE_PARSE(LegacySSHStoreConfig, legacySSHStoreConfig, LEGACY_SSH_STORE_CONFIG_FIELDS)
static LegacySSHStoreConfigT<config::JustValue> legacySSHStoreConfigDefaults()
{
return {
.remoteProgram = {{"nix-store"}},
.maxConnections = {1},
};
}
MAKE_APPLY_PARSE(LegacySSHStoreConfig, legacySSHStoreConfig, LEGACY_SSH_STORE_CONFIG_FIELDS)
config::SettingDescriptionMap LegacySSHStoreConfig::descriptions()
{
config::SettingDescriptionMap ret;
ret.merge(StoreConfig::descriptions());
ret.merge(CommonSSHStoreConfig::descriptions());
ret.merge(RemoteStoreConfig::descriptions());
{
constexpr auto & descriptions = legacySSHStoreConfigDescriptions;
auto defaults = legacySSHStoreConfigDefaults();
ret.merge(decltype(ret){
LEGACY_SSH_STORE_CONFIG_FIELDS(DESC_ROW)
});
}
return ret;
}
LegacySSHStore::Config::LegacySSHStoreConfig(
std::string_view scheme,
std::string_view authority,
const Params & params)
: StoreConfig(params)
, CommonSSHStoreConfig(scheme, authority, params)
const StoreReference::Params & params)
: Store::Config{params}
, CommonSSHStoreConfig{scheme, authority, params}
, LegacySSHStoreConfigT<config::JustValue>{legacySSHStoreConfigApplyParse(params)}
{
#ifndef _WIN32
if (auto * p = get(params, "log-fd")) {
logFD = p->get<decltype(logFD)>();
}
#endif
}
std::string LegacySSHStoreConfig::doc()
{
return
@ -38,23 +96,19 @@ struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection
bool good = true;
};
LegacySSHStore::LegacySSHStore(
std::string_view scheme,
std::string_view host,
const Params & params)
: StoreConfig(params)
, CommonSSHStoreConfig(scheme, host, params)
, LegacySSHStoreConfig(scheme, host, params)
, Store(params)
LegacySSHStore::LegacySSHStore(ref<const Config> config)
: Store{*config}
, config{config}
, connections(make_ref<Pool<Connection>>(
std::max(1, (int) maxConnections),
std::max(1, (int) config->maxConnections),
[this]() { return openConnection(); },
[](const ref<Connection> & r) { return r->good; }
))
, master(createSSHMaster(
, master(config->createSSHMaster(
// Use SSH master only if using more than 1 connection.
connections->capacity() > 1,
logFD))
config->logFD))
{
}
@ -62,16 +116,16 @@ LegacySSHStore::LegacySSHStore(
ref<LegacySSHStore::Connection> LegacySSHStore::openConnection()
{
auto conn = make_ref<Connection>();
Strings command = remoteProgram.get();
Strings command = config->remoteProgram.get();
command.push_back("--serve");
command.push_back("--write");
if (remoteStore.get() != "") {
if (config->remoteStore.get() != "") {
command.push_back("--store");
command.push_back(remoteStore.get());
command.push_back(config->remoteStore.get());
}
conn->sshConn = master.startCommand(std::move(command), std::list{extraSshArgs});
if (connPipeSize) {
conn->sshConn->trySetBufferSize(*connPipeSize);
conn->sshConn = master.startCommand(std::move(command), std::list{config->extraSshArgs});
if (config->connPipeSize) {
conn->sshConn->trySetBufferSize(*config->connPipeSize);
}
conn->to = FdSink(conn->sshConn->in.get());
conn->from = FdSource(conn->sshConn->out.get());
@ -80,7 +134,7 @@ ref<LegacySSHStore::Connection> LegacySSHStore::openConnection()
TeeSource tee(conn->from, saved);
try {
conn->remoteVersion = ServeProto::BasicClientConnection::handshake(
conn->to, tee, SERVE_PROTOCOL_VERSION, host);
conn->to, tee, SERVE_PROTOCOL_VERSION, config->host);
} catch (SerialisationError & e) {
// in.close(): Don't let the remote block on us not writing.
conn->sshConn->in.close();
@ -89,9 +143,9 @@ ref<LegacySSHStore::Connection> LegacySSHStore::openConnection()
tee.drainInto(nullSink);
}
throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'",
host, chomp(saved.s));
config->host, chomp(saved.s));
} catch (EndOfFile & e) {
throw Error("cannot connect to '%1%'", host);
throw Error("cannot connect to '%1%'", config->host);
}
return conn;
@ -100,7 +154,7 @@ ref<LegacySSHStore::Connection> LegacySSHStore::openConnection()
std::string LegacySSHStore::getUri()
{
return *uriSchemes().begin() + "://" + host;
return *Config::uriSchemes().begin() + "://" + config->host;
}
std::map<StorePath, UnkeyedValidPathInfo> LegacySSHStore::queryPathInfosUncached(
@ -111,7 +165,7 @@ std::map<StorePath, UnkeyedValidPathInfo> LegacySSHStore::queryPathInfosUncached
/* No longer support missing NAR hash */
assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4);
debug("querying remote host '%s' for info on '%s'", host, concatStringsSep(", ", printStorePathSet(paths)));
debug("querying remote host '%s' for info on '%s'", config->host, concatStringsSep(", ", printStorePathSet(paths)));
auto infos = conn->queryPathInfos(*this, paths);
@ -151,7 +205,7 @@ void LegacySSHStore::queryPathInfoUncached(const StorePath & path,
void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs)
{
debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host);
debug("adding path '%s' to remote host '%s'", printStorePath(info.path), config->host);
auto conn(connections->get());
@ -178,7 +232,7 @@ void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source,
conn->to.flush();
if (readInt(conn->from) != 1)
throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host);
throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), config->host);
} else {
@ -390,12 +444,17 @@ LegacySSHStore::ConnectionStats LegacySSHStore::getConnectionStats()
* The legacy ssh protocol doesn't support checking for trusted-user.
* Try using ssh-ng:// instead if you want to know.
*/
std::optional<TrustedFlag> isTrustedClient()
std::optional<TrustedFlag> LegacySSHStore::isTrustedClient()
{
return std::nullopt;
}
static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regLegacySSHStore;
ref<Store> LegacySSHStore::Config::openStore() const {
return make_ref<LegacySSHStore>(ref{shared_from_this()});
}
static RegisterStoreImplementation<LegacySSHStore::Config> regLegacySSHStore;
}

View File

@ -2,17 +2,27 @@
#include "nix/store/globals.hh"
#include "nix/store/nar-info-disk-cache.hh"
#include "nix/util/signals.hh"
#include "nix/store/store-registration.hh"
#include <atomic>
namespace nix {
config::SettingDescriptionMap LocalBinaryCacheStoreConfig::descriptions()
{
config::SettingDescriptionMap ret;
ret.merge(StoreConfig::descriptions());
ret.merge(BinaryCacheStoreConfig::descriptions());
return ret;
}
LocalBinaryCacheStoreConfig::LocalBinaryCacheStoreConfig(
std::string_view scheme,
PathView binaryCacheDir,
const Params & params)
: StoreConfig(params)
, BinaryCacheStoreConfig(params)
const StoreReference::Params & params)
: Store::Config{params}
, BinaryCacheStoreConfig{*this, params}
, binaryCacheDir(binaryCacheDir)
{
}
@ -26,29 +36,26 @@ std::string LocalBinaryCacheStoreConfig::doc()
}
struct LocalBinaryCacheStore : virtual LocalBinaryCacheStoreConfig, virtual BinaryCacheStore
struct LocalBinaryCacheStore :
virtual BinaryCacheStore
{
/**
* @param binaryCacheDir `file://` is a short-hand for `file:///`
* for now.
*/
LocalBinaryCacheStore(
std::string_view scheme,
PathView binaryCacheDir,
const Params & params)
: StoreConfig(params)
, BinaryCacheStoreConfig(params)
, LocalBinaryCacheStoreConfig(scheme, binaryCacheDir, params)
, Store(params)
, BinaryCacheStore(params)
using Config = LocalBinaryCacheStoreConfig;
ref<const Config> config;
LocalBinaryCacheStore(ref<const Config> config)
: Store{*config}
, BinaryCacheStore{*config}
, config{config}
{
init();
}
void init() override;
std::string getUri() override
{
return "file://" + binaryCacheDir;
return "file://" + config->binaryCacheDir;
}
protected:
@ -59,7 +66,7 @@ protected:
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) override
{
auto path2 = binaryCacheDir + "/" + path;
auto path2 = config->binaryCacheDir + "/" + path;
static std::atomic<int> counter{0};
Path tmp = fmt("%s.tmp.%d.%d", path2, getpid(), ++counter);
AutoDelete del(tmp, false);
@ -72,7 +79,7 @@ protected:
void getFile(const std::string & path, Sink & sink) override
{
try {
readFile(binaryCacheDir + "/" + path, sink);
readFile(config->binaryCacheDir + "/" + path, sink);
} catch (SysError & e) {
if (e.errNo == ENOENT)
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path);
@ -84,7 +91,7 @@ protected:
{
StorePathSet paths;
for (auto & entry : std::filesystem::directory_iterator{binaryCacheDir}) {
for (auto & entry : std::filesystem::directory_iterator{config->binaryCacheDir}) {
checkInterrupt();
auto name = entry.path().filename().string();
if (name.size() != 40 ||
@ -106,17 +113,17 @@ protected:
void LocalBinaryCacheStore::init()
{
createDirs(binaryCacheDir + "/nar");
createDirs(binaryCacheDir + "/" + realisationsPrefix);
if (writeDebugInfo)
createDirs(binaryCacheDir + "/debuginfo");
createDirs(binaryCacheDir + "/log");
createDirs(config->binaryCacheDir + "/nar");
createDirs(config->binaryCacheDir + "/" + realisationsPrefix);
if (config->writeDebugInfo)
createDirs(config->binaryCacheDir + "/debuginfo");
createDirs(config->binaryCacheDir + "/log");
BinaryCacheStore::init();
}
bool LocalBinaryCacheStore::fileExists(const std::string & path)
{
return pathExists(binaryCacheDir + "/" + path);
return pathExists(config->binaryCacheDir + "/" + path);
}
std::set<std::string> LocalBinaryCacheStoreConfig::uriSchemes()
@ -127,6 +134,10 @@ std::set<std::string> LocalBinaryCacheStoreConfig::uriSchemes()
return {"file"};
}
static RegisterStoreImplementation<LocalBinaryCacheStore, LocalBinaryCacheStoreConfig> regLocalBinaryCacheStore;
ref<Store> LocalBinaryCacheStoreConfig::openStore() const {
return make_ref<LocalBinaryCacheStore>(ref{shared_from_this()});
}
static RegisterStoreImplementation<LocalBinaryCacheStore::Config> regLocalBinaryCacheStore;
}

View File

@ -1,3 +1,4 @@
#include "nix/util/json-utils.hh"
#include "nix/util/archive.hh"
#include "nix/util/posix-source-accessor.hh"
#include "nix/store/store-api.hh"
@ -5,25 +6,108 @@
#include "nix/store/globals.hh"
#include "nix/util/compression.hh"
#include "nix/store/derivations.hh"
#include "nix/store/config-parse-impl.hh"
namespace nix {
LocalFSStoreConfig::LocalFSStoreConfig(PathView rootDir, const Params & params)
: StoreConfig(params)
// Default `?root` from `rootDir` if non set
// FIXME don't duplicate description once we don't have root setting
, rootDir{
this,
!rootDir.empty() && params.count("root") == 0
? (std::optional<Path>{rootDir})
: std::nullopt,
"root",
"Directory prefixed to all other paths."}
constexpr static const LocalFSStoreConfigT<config::SettingInfo> localFSStoreConfigDescriptions = {
.rootDir = {
.name = "root",
.description = "Directory prefixed to all other paths.",
},
.stateDir = {
.name = "state",
.description = "Directory where Nix will store state.",
},
.logDir = {
.name = "log",
.description = "directory where Nix will store log files.",
},
.realStoreDir{
.name = "real",
.description = "Physical path of the Nix store.",
},
};
#define LOCAL_FS_STORE_CONFIG_FIELDS(X) \
X(rootDir), \
X(stateDir), \
X(logDir), \
X(realStoreDir),
MAKE_PARSE(LocalFSStoreConfig, localFSStoreConfig, LOCAL_FS_STORE_CONFIG_FIELDS)
/**
* @param rootDir Fallback if not in `params`
*/
static LocalFSStoreConfigT<config::JustValue> localFSStoreConfigDefaults(
const Path & storeDir,
const std::optional<Path> & rootDir)
{
return {
.rootDir = {std::nullopt},
.stateDir = {rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir},
.logDir = {rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir},
.realStoreDir = {rootDir ? *rootDir + "/nix/store" : storeDir},
};
}
static LocalFSStoreConfigT<config::JustValue> localFSStoreConfigApplyParse(
const Path & storeDir,
LocalFSStoreConfigT<config::OptValue> parsed)
{
auto defaults = localFSStoreConfigDefaults(
storeDir,
parsed.rootDir.optValue.value_or(std::nullopt));
return {LOCAL_FS_STORE_CONFIG_FIELDS(APPLY_ROW)};
}
config::SettingDescriptionMap LocalFSStoreConfig::descriptions()
{
constexpr auto & descriptions = localFSStoreConfigDescriptions;
auto defaults = localFSStoreConfigDefaults(settings.nixStore, std::nullopt);
return {
LOCAL_FS_STORE_CONFIG_FIELDS(DESC_ROW)
};
}
LocalFSStore::Config::LocalFSStoreConfig(
const Store::Config & storeConfig,
const StoreReference::Params & params)
: LocalFSStoreConfigT<config::JustValue>{
localFSStoreConfigApplyParse(
storeConfig.storeDir,
localFSStoreConfigParse(params))}
, storeConfig{storeConfig}
{
}
LocalFSStore::LocalFSStore(const Params & params)
: Store(params)
static LocalFSStoreConfigT<config::OptValue> applyAuthority(
LocalFSStoreConfigT<config::OptValue> parsed,
PathView rootDir)
{
if (!rootDir.empty())
parsed.rootDir = {.optValue = {Path{rootDir}}};
return parsed;
}
LocalFSStore::Config::LocalFSStoreConfig(
const Store::Config & storeConfig,
PathView rootDir,
const StoreReference::Params & params)
: LocalFSStoreConfigT<config::JustValue>{
localFSStoreConfigApplyParse(
storeConfig.storeDir,
applyAuthority(
localFSStoreConfigParse(params),
rootDir))}
, storeConfig{storeConfig}
{
}
LocalFSStore::LocalFSStore(const Config & config)
: Store{static_cast<const Store::Config &>(*this)}
, config{config}
{
}
@ -33,7 +117,7 @@ struct LocalStoreAccessor : PosixSourceAccessor
bool requireValidPath;
LocalStoreAccessor(ref<LocalFSStore> store, bool requireValidPath)
: PosixSourceAccessor(std::filesystem::path{store->realStoreDir.get()})
: PosixSourceAccessor(std::filesystem::path{store->config.realStoreDir.get()})
, store(store)
, requireValidPath(requireValidPath)
{
@ -104,8 +188,8 @@ std::optional<std::string> LocalFSStore::getBuildLogExact(const StorePath & path
Path logPath =
j == 0
? fmt("%s/%s/%s/%s", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2))
: fmt("%s/%s/%s", logDir, drvsLogDir, baseName);
? fmt("%s/%s/%s/%s", config.logDir.get(), drvsLogDir, baseName.substr(0, 2), baseName.substr(2))
: fmt("%s/%s/%s", config.logDir.get(), drvsLogDir, baseName);
Path logBz2Path = logPath + ".bz2";
if (pathExists(logPath))

View File

@ -1,12 +1,110 @@
#include <regex>
#include "nix/store/local-overlay-store.hh"
#include "nix/util/callback.hh"
#include "nix/store/realisation.hh"
#include "nix/util/processes.hh"
#include "nix/util/url.hh"
#include <regex>
#include "nix/store/store-api.hh"
#include "nix/store/store-registration.hh"
#include "nix/store/config-parse-impl.hh"
namespace nix {
static LocalOverlayStoreConfigT<config::SettingInfo> localOverlayStoreConfigDescriptions = {
.lowerStoreConfig{
.name = "lower-store",
.description = R"(
[Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly).
Must be a store with a store dir on the file system.
Must be used as OverlayFS lower layer for this store's store dir.
)",
// It's not actually machine-specific, but we don't yet have a
// `to_json` for `StoreConfig`.
.documentDefault = false,
},
.upperLayer{
.name = "upper-layer",
.description = R"(
Directory containing the OverlayFS upper layer for this store's store dir.
)",
},
.checkMount{
.name = "check-mount",
.description = R"(
Check that the overlay filesystem is correctly mounted.
Nix does not manage the overlayfs mount point itself, but the correct
functioning of the overlay store does depend on this mount point being set up
correctly. Rather than just assume this is the case, check that the lowerdir
and upperdir options are what we expect them to be. This check is on by
default, but can be disabled if needed.
)",
},
.remountHook{
.name = "remount-hook",
.description = R"(
Script or other executable to run when overlay filesystem needs remounting.
This is occasionally necessary when deleting a store path that exists in both upper and lower layers.
In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly
is the only way to perform the deletion without creating a "whiteout".
However this causes the OverlayFS kernel data structures to get out-of-sync,
and can lead to 'stale file handle' errors; remounting solves the problem.
The store directory is passed as an argument to the invoked executable.
)",
},
};
#define LOCAL_OVERLAY_STORE_CONFIG_FIELDS(X) \
X(lowerStoreConfig), \
X(upperLayer), \
X(checkMount), \
X(remountHook),
MAKE_PARSE(LocalOverlayStoreConfig, localOverlayStoreConfig, LOCAL_OVERLAY_STORE_CONFIG_FIELDS)
static LocalOverlayStoreConfigT<config::JustValue> localOverlayStoreConfigDefaults()
{
return {
.lowerStoreConfig = {make_ref<LocalStore::Config>(StoreReference::Params{})},
.upperLayer = {""},
.checkMount = {true},
.remountHook = {""},
};
}
MAKE_APPLY_PARSE(LocalOverlayStoreConfig, localOverlayStoreConfig, LOCAL_OVERLAY_STORE_CONFIG_FIELDS)
config::SettingDescriptionMap LocalOverlayStoreConfig::descriptions()
{
config::SettingDescriptionMap ret;
ret.merge(StoreConfig::descriptions());
ret.merge(LocalFSStoreConfig::descriptions());
ret.merge(LocalStoreConfig::descriptions());
{
constexpr auto & descriptions = localOverlayStoreConfigDescriptions;
auto defaults = localOverlayStoreConfigDefaults();
ret.merge(decltype(ret){
LOCAL_OVERLAY_STORE_CONFIG_FIELDS(DESC_ROW)
});
}
return ret;
}
LocalOverlayStore::Config::LocalOverlayStoreConfig(
std::string_view scheme,
std::string_view authority,
const StoreReference::Params & params)
: LocalStore::Config(scheme, authority, params)
, LocalOverlayStoreConfigT<config::JustValue>{localOverlayStoreConfigApplyParse(params)}
{
}
std::string LocalOverlayStoreConfig::doc()
{
return
@ -14,25 +112,32 @@ std::string LocalOverlayStoreConfig::doc()
;
}
Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) {
ref<Store> LocalOverlayStoreConfig::openStore() const
{
return make_ref<LocalOverlayStore>(ref{
std::dynamic_pointer_cast<const LocalOverlayStoreConfig>(shared_from_this())
});
}
Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) const
{
return upperLayer + "/" + path.to_string();
}
LocalOverlayStore::LocalOverlayStore(std::string_view scheme, PathView path, const Params & params)
: StoreConfig(params)
, LocalFSStoreConfig(path, params)
, LocalStoreConfig(params)
, LocalOverlayStoreConfig(scheme, path, params)
, Store(params)
, LocalFSStore(params)
, LocalStore(params)
, lowerStore(openStore(percentDecode(lowerStoreUri.get())).dynamic_pointer_cast<LocalFSStore>())
LocalOverlayStore::LocalOverlayStore(ref<const Config> config)
: Store{*config}
, LocalFSStore{*config}
, LocalStore{static_cast<ref<const LocalStore::Config>>(config)}
, config{config}
, lowerStore(config->lowerStoreConfig.value->openStore().dynamic_pointer_cast<LocalFSStore>())
{
if (checkMount.get()) {
if (config->checkMount.get()) {
std::smatch match;
std::string mountInfo;
auto mounts = readFile(std::filesystem::path{"/proc/self/mounts"});
auto regex = std::regex(R"((^|\n)overlay )" + realStoreDir.get() + R"( .*(\n|$))");
auto regex = std::regex(R"((^|\n)overlay )" + config->realStoreDir.get() + R"( .*(\n|$))");
// Mount points can be stacked, so there might be multiple matching entries.
// Loop until the last match, which will be the current state of the mount point.
@ -45,13 +150,13 @@ LocalOverlayStore::LocalOverlayStore(std::string_view scheme, PathView path, con
return std::regex_search(mountInfo, std::regex("\\b" + option + "=" + value + "( |,)"));
};
auto expectedLowerDir = lowerStore->realStoreDir.get();
if (!checkOption("lowerdir", expectedLowerDir) || !checkOption("upperdir", upperLayer)) {
auto expectedLowerDir = lowerStore->config.realStoreDir.get();
if (!checkOption("lowerdir", expectedLowerDir) || !checkOption("upperdir", config->upperLayer)) {
debug("expected lowerdir: %s", expectedLowerDir);
debug("expected upperdir: %s", upperLayer);
debug("expected upperdir: %s", config->upperLayer);
debug("actual mount: %s", mountInfo);
throw Error("overlay filesystem '%s' mounted incorrectly",
realStoreDir.get());
config->realStoreDir.get());
}
}
}
@ -201,14 +306,14 @@ void LocalOverlayStore::collectGarbage(const GCOptions & options, GCResults & re
void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed)
{
auto mergedDir = realStoreDir.get() + "/";
auto mergedDir = config->realStoreDir.get() + "/";
if (path.substr(0, mergedDir.length()) != mergedDir) {
warn("local-overlay: unexpected gc path '%s' ", path);
return;
}
StorePath storePath = {path.substr(mergedDir.length())};
auto upperPath = toUpperPath(storePath);
auto upperPath = config->toUpperPath(storePath);
if (pathExists(upperPath)) {
debug("upper exists: %s", path);
@ -257,7 +362,7 @@ LocalStore::VerificationResult LocalOverlayStore::verifyAllValidPaths(RepairFlag
StorePathSet done;
auto existsInStoreDir = [&](const StorePath & storePath) {
return pathExists(realStoreDir.get() + "/" + storePath.to_string());
return pathExists(config->realStoreDir.get() + "/" + storePath.to_string());
};
bool errors = false;
@ -277,16 +382,16 @@ void LocalOverlayStore::remountIfNecessary()
{
if (!_remountRequired) return;
if (remountHook.get().empty()) {
warn("'%s' needs remounting, set remount-hook to do this automatically", realStoreDir.get());
if (config->remountHook.get().empty()) {
warn("'%s' needs remounting, set remount-hook to do this automatically", config->realStoreDir.get());
} else {
runProgram(remountHook, false, {realStoreDir});
runProgram(config->remountHook, false, {config->realStoreDir});
}
_remountRequired = false;
}
static RegisterStoreImplementation<LocalOverlayStore, LocalOverlayStoreConfig> regLocalOverlayStore;
static RegisterStoreImplementation<LocalOverlayStore::Config> regLocalOverlayStore;
}

View File

@ -17,6 +17,9 @@
#include "nix/util/posix-source-accessor.hh"
#include "nix/store/keys.hh"
#include "nix/util/users.hh"
#include "nix/store/config-parse-impl.hh"
#include "nix/store/store-open.hh"
#include "nix/store/store-registration.hh"
#include <iostream>
#include <algorithm>
@ -59,15 +62,72 @@
namespace nix {
LocalStoreConfig::LocalStoreConfig(
constexpr static const LocalStoreConfigT<config::SettingInfo> localStoreConfigDescriptions = {
.requireSigs = {
.name = "require-sigs",
.description = "Whether store paths copied into this store should have a trusted signature.",
},
.readOnly = {
.name = "read-only",
.description = R"(
Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem.
Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem.
Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set.
> **Warning**
> Do not use this unless the filesystem is read-only.
>
> Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process.
> While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it.
)",
},
};
#define LOCAL_STORE_CONFIG_FIELDS(X) \
X(requireSigs), \
X(readOnly),
MAKE_PARSE(LocalStoreConfig, localStoreConfig, LOCAL_STORE_CONFIG_FIELDS)
static LocalStoreConfigT<config::JustValue> localStoreConfigDefaults()
{
return {
.requireSigs = {settings.requireSigs},
.readOnly = {false},
};
}
MAKE_APPLY_PARSE(LocalStoreConfig, localStoreConfig, LOCAL_STORE_CONFIG_FIELDS)
config::SettingDescriptionMap LocalStoreConfig::descriptions()
{
config::SettingDescriptionMap ret;
ret.merge(StoreConfig::descriptions());
ret.merge(LocalFSStoreConfig::descriptions());
{
constexpr auto & descriptions = localStoreConfigDescriptions;
auto defaults = localStoreConfigDefaults();
ret.merge(decltype(ret){
LOCAL_STORE_CONFIG_FIELDS(DESC_ROW)
});
}
return ret;
}
LocalStore::Config::LocalStoreConfig(
std::string_view scheme,
std::string_view authority,
const Params & params)
: StoreConfig(params)
, LocalFSStoreConfig(authority, params)
const StoreReference::Params & params)
: Store::Config(params)
, LocalFSStore::Config(*this, authority, params)
, LocalStoreConfigT<config::JustValue>{localStoreConfigApplyParse(params)}
{
}
LocalStoreConfig::LocalStoreConfig(const LocalStoreConfig &) = default;
std::string LocalStoreConfig::doc()
{
return
@ -75,6 +135,11 @@ std::string LocalStoreConfig::doc()
;
}
ref<Store> LocalStore::Config::openStore() const
{
return make_ref<LocalStore>(ref{shared_from_this()});
}
struct LocalStore::State::Stmts {
/* Some precompiled SQLite statements. */
SQLiteStmt RegisterValidPath;
@ -97,38 +162,33 @@ struct LocalStore::State::Stmts {
SQLiteStmt AddRealisationReference;
};
LocalStore::LocalStore(
std::string_view scheme,
PathView path,
const Params & params)
: StoreConfig(params)
, LocalFSStoreConfig(path, params)
, LocalStoreConfig(scheme, path, params)
, Store(params)
, LocalFSStore(params)
, dbDir(stateDir + "/db")
, linksDir(realStoreDir + "/.links")
LocalStore::LocalStore(ref<const Config> config)
: Store{*config}
, LocalFSStore{*config}
, config{config}
, dbDir(config->stateDir + "/db")
, linksDir(config->realStoreDir + "/.links")
, reservedPath(dbDir + "/reserved")
, schemaPath(dbDir + "/schema")
, tempRootsDir(stateDir + "/temproots")
, tempRootsDir(config->stateDir + "/temproots")
, fnTempRoots(fmt("%s/%d", tempRootsDir, getpid()))
{
auto state(_state.lock());
state->stmts = std::make_unique<State::Stmts>();
/* Create missing state directories if they don't already exist. */
createDirs(realStoreDir.get());
if (readOnly) {
createDirs(config->realStoreDir.get());
if (config->readOnly) {
experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore);
} else {
makeStoreWritable();
}
createDirs(linksDir);
Path profilesDir = stateDir + "/profiles";
Path profilesDir = config->stateDir + "/profiles";
createDirs(profilesDir);
createDirs(tempRootsDir);
createDirs(dbDir);
Path gcRootsDir = stateDir + "/gcroots";
Path gcRootsDir = config->stateDir + "/gcroots";
if (!pathExists(gcRootsDir)) {
createDirs(gcRootsDir);
createSymlink(profilesDir, gcRootsDir + "/profiles");
@ -136,7 +196,7 @@ LocalStore::LocalStore(
for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) {
createDirs(perUserDir);
if (!readOnly) {
if (!config->readOnly) {
// Skip chmod call if the directory already has the correct permissions (0755).
// This is to avoid failing when the executing user lacks permissions to change the directory's permissions
// even if it would be no-op.
@ -153,16 +213,16 @@ LocalStore::LocalStore(
struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
if (!gr)
printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup);
else if (!readOnly) {
else if (!config->readOnly) {
struct stat st;
if (stat(realStoreDir.get().c_str(), &st))
throw SysError("getting attributes of path '%1%'", realStoreDir);
if (stat(config->realStoreDir.get().c_str(), &st))
throw SysError("getting attributes of path '%1%'", config->realStoreDir);
if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) {
if (chown(realStoreDir.get().c_str(), 0, gr->gr_gid) == -1)
throw SysError("changing ownership of path '%1%'", realStoreDir);
if (chmod(realStoreDir.get().c_str(), perm) == -1)
throw SysError("changing permissions on path '%1%'", realStoreDir);
if (chown(config->realStoreDir.get().c_str(), 0, gr->gr_gid) == -1)
throw SysError("changing ownership of path '%1%'", config->realStoreDir);
if (chmod(config->realStoreDir.get().c_str(), perm) == -1)
throw SysError("changing permissions on path '%1%'", config->realStoreDir);
}
}
}
@ -170,7 +230,7 @@ LocalStore::LocalStore(
/* Ensure that the store and its parents are not symlinks. */
if (!settings.allowSymlinkedStore) {
std::filesystem::path path = realStoreDir.get();
std::filesystem::path path = config->realStoreDir.get();
std::filesystem::path root = path.root_path();
while (path != root) {
if (std::filesystem::is_symlink(path))
@ -217,12 +277,12 @@ LocalStore::LocalStore(
/* Acquire the big fat lock in shared mode to make sure that no
schema upgrade is in progress. */
if (!readOnly) {
if (!config->readOnly) {
Path globalLockPath = dbDir + "/big-lock";
globalLock = openLockFile(globalLockPath.c_str(), true);
}
if (!readOnly && !lockFile(globalLock.get(), ltRead, false)) {
if (!config->readOnly && !lockFile(globalLock.get(), ltRead, false)) {
printInfo("waiting for the big Nix store lock...");
lockFile(globalLock.get(), ltRead, true);
}
@ -230,7 +290,7 @@ LocalStore::LocalStore(
/* Check the current database schema and if necessary do an
upgrade. */
int curSchema = getSchema();
if (readOnly && curSchema < nixSchemaVersion) {
if (config->readOnly && curSchema < nixSchemaVersion) {
debug("current schema version: %d", curSchema);
debug("supported schema version: %d", nixSchemaVersion);
throw Error(curSchema == 0 ?
@ -378,15 +438,9 @@ LocalStore::LocalStore(
}
LocalStore::LocalStore(const Params & params)
: LocalStore("local", "", params)
{
}
AutoCloseFD LocalStore::openGCLock()
{
Path fnGCLock = stateDir + "/gc.lock";
Path fnGCLock = config->stateDir + "/gc.lock";
auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT
#ifndef _WIN32
| O_CLOEXEC
@ -452,17 +506,17 @@ int LocalStore::getSchema()
void LocalStore::openDB(State & state, bool create)
{
if (create && readOnly) {
if (create && config->readOnly) {
throw Error("cannot create database while in read-only mode");
}
if (access(dbDir.c_str(), R_OK | (readOnly ? 0 : W_OK)))
if (access(dbDir.c_str(), R_OK | (config->readOnly ? 0 : W_OK)))
throw SysError("Nix database directory '%1%' is not writable", dbDir);
/* Open the Nix database. */
std::string dbPath = dbDir + "/db.sqlite";
auto & db(state.db);
auto openMode = readOnly ? SQLiteOpenMode::Immutable
auto openMode = config->readOnly ? SQLiteOpenMode::Immutable
: create ? SQLiteOpenMode::Normal
: SQLiteOpenMode::NoCreate;
state.db = SQLite(dbPath, openMode);
@ -575,12 +629,12 @@ void LocalStore::makeStoreWritable()
if (!isRootUser()) return;
/* Check if /nix/store is on a read-only mount. */
struct statvfs stat;
if (statvfs(realStoreDir.get().c_str(), &stat) != 0)
if (statvfs(config->realStoreDir.get().c_str(), &stat) != 0)
throw SysError("getting info about the Nix store mount point");
if (stat.f_flag & ST_RDONLY) {
if (mount(0, realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
throw SysError("remounting %1% writable", realStoreDir);
if (mount(0, config->realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
throw SysError("remounting %1% writable", config->realStoreDir);
}
#endif
}
@ -920,7 +974,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
for (auto & sub : getDefaultSubstituters()) {
if (remaining.empty()) break;
if (sub->storeDir != storeDir) continue;
if (!sub->wantMassQuery) continue;
if (!sub->resolvedSubstConfig.wantMassQuery) continue;
auto valid = sub->queryValidPaths(remaining);
@ -1032,12 +1086,12 @@ const PublicKeys & LocalStore::getPublicKeys()
bool LocalStore::pathInfoIsUntrusted(const ValidPathInfo & info)
{
return requireSigs && !info.checkSignatures(*this, getPublicKeys());
return config->requireSigs && !info.checkSignatures(*this, getPublicKeys());
}
bool LocalStore::realisationIsUntrusted(const Realisation & realisation)
{
return requireSigs && !realisation.checkSignatures(getPublicKeys());
return config->requireSigs && !realisation.checkSignatures(getPublicKeys());
}
void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
@ -1334,7 +1388,7 @@ std::pair<std::filesystem::path, AutoCloseFD> LocalStore::createTempDirInStore()
/* There is a slight possibility that `tmpDir' gets deleted by
the GC between createTempDir() and when we acquire a lock on it.
We'll repeat until 'tmpDir' exists and we've locked it. */
tmpDirFn = createTempDir(realStoreDir, "tmp");
tmpDirFn = createTempDir(config->realStoreDir, "tmp");
tmpDirFd = openDirectory(tmpDirFn);
if (!tmpDirFd) {
continue;
@ -1475,7 +1529,7 @@ LocalStore::VerificationResult LocalStore::verifyAllValidPaths(RepairFlag repair
database and the filesystem) in the loop below, in order to catch
invalid states.
*/
for (auto & i : std::filesystem::directory_iterator{realStoreDir.to_string()}) {
for (auto & i : std::filesystem::directory_iterator{config->realStoreDir.get()}) {
checkInterrupt();
try {
storePathsInStoreDir.insert({i.path().filename().string()});
@ -1664,7 +1718,7 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log)
auto baseName = drvPath.to_string();
auto logPath = fmt("%s/%s/%s/%s.bz2", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2));
auto logPath = fmt("%s/%s/%s/%s.bz2", config->logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2));
if (pathExists(logPath)) return;
@ -1682,6 +1736,6 @@ std::optional<std::string> LocalStore::getVersion()
return nixVersion;
}
static RegisterStoreImplementation<LocalStore, LocalStoreConfig> regLocalStore;
static RegisterStoreImplementation<LocalStore::Config> regLocalStore;
} // namespace nix

View File

@ -1,6 +1,6 @@
#include "nix/store/machines.hh"
#include "nix/store/globals.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include <algorithm>
@ -71,8 +71,8 @@ StoreReference Machine::completeStoreReference() const
auto * generic = std::get_if<StoreReference::Specified>(&storeUri.variant);
if (generic && generic->scheme == "ssh") {
storeUri.params["max-connections"] = "1";
storeUri.params["log-fd"] = "4";
storeUri.params["max-connections"] = 1;
storeUri.params["log-fd"] = 4;
}
if (generic && (generic->scheme == "ssh" || generic->scheme == "ssh-ng")) {
@ -84,14 +84,10 @@ StoreReference Machine::completeStoreReference() const
{
auto & fs = storeUri.params["system-features"];
auto append = [&](auto feats) {
for (auto & f : feats) {
if (fs.size() > 0) fs += ' ';
fs += f;
}
};
append(supportedFeatures);
append(mandatoryFeatures);
if (!fs.is_array()) fs = nlohmann::json::array();
auto features = supportedFeatures;
features.insert(supportedFeatures.begin(), supportedFeatures.end());
for (auto & feat : features) fs += feat;
}
return storeUri;

View File

@ -264,6 +264,7 @@ sources = files(
'builtins/unpack-channel.cc',
'common-protocol.cc',
'common-ssh-store-config.cc',
'config-parse.cc',
'content-address.cc',
'daemon.cc',
'derivations.cc',
@ -313,6 +314,8 @@ sources = files(
'ssh-store.cc',
'ssh.cc',
'store-api.cc',
'store-dir-config.cc',
'store-registration.cc',
'store-reference.cc',
'uds-remote-store.cc',
'worker-protocol-connection.cc',

View File

@ -4,7 +4,7 @@
#include "nix/store/parsed-derivations.hh"
#include "nix/store/derivation-options.hh"
#include "nix/store/globals.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/util/thread-pool.hh"
#include "nix/store/realisation.hh"
#include "nix/util/topo-sort.hh"

View File

@ -1,18 +0,0 @@
R"(
**Store URL format**: `mounted-ssh-ng://[username@]hostname`
Experimental store type that allows full access to a Nix store on a remote machine,
and additionally requires that store be mounted in the local file system.
The mounting of that store is not managed by Nix, and must by managed manually.
It could be accomplished with SSHFS or NFS, for example.
The local file system is used to optimize certain operations.
For example, rather than serializing Nix archives and sending over the Nix channel,
we can directly access the file system data via the mount-point.
The local file system is also used to make certain operations possible that wouldn't otherwise be.
For example, persistent GC roots can be created if they reside on the same file system as the remote store:
the remote side will create the symlinks necessary to avoid race conditions.
)"

View File

@ -101,7 +101,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
/* HFS/macOS has some undocumented security feature disabling hardlinking for
special files within .app dirs. Known affected paths include
*.app/Contents/{PkgInfo,Resources/\*.lproj,_CodeSignature} and .DS_Store.
See https://github.com/NixOS/nix/issues/1443 and
See https://github.com/NixOS/nix/issues/1443 and
https://github.com/NixOS/nix/pull/2230 for more discussion. */
if (std::regex_search(path, std::regex("\\.app/Contents/.+$")))
@ -216,14 +216,14 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
the store itself (we don't want or need to mess with its
permissions). */
const Path dirOfPath(dirOf(path));
bool mustToggle = dirOfPath != realStoreDir.get();
bool mustToggle = dirOfPath != config->realStoreDir.get();
if (mustToggle) makeWritable(dirOfPath);
/* When we're done, make the directory read-only again and reset
its timestamp back to 0. */
MakeReadOnly makeReadOnly(mustToggle ? dirOfPath : "");
std::filesystem::path tempLink = fmt("%1%/.tmp-link-%2%-%3%", realStoreDir, getpid(), rand());
std::filesystem::path tempLink = fmt("%1%/.tmp-link-%2%-%3%", config->realStoreDir, getpid(), rand());
try {
std::filesystem::create_hard_link(linkPath, tempLink);
@ -285,7 +285,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
if (!isValidPath(i)) continue; /* path was GC'ed, probably */
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", printStorePath(i)));
optimisePath_(&act, stats, realStoreDir + "/" + std::string(i.to_string()), inodeHash, NoRepair);
optimisePath_(&act, stats, config->realStoreDir + "/" + std::string(i.to_string()), inodeHash, NoRepair);
}
done++;
act.progress(done, paths.size());

View File

@ -75,7 +75,7 @@ StorePath StorePath::random(std::string_view name)
return StorePath(Hash::random(HashAlgorithm::SHA1), name);
}
StorePath StoreDirConfig::parseStorePath(std::string_view path) const
StorePath MixStoreDirMethods::parseStorePath(std::string_view path) const
{
// On Windows, `/nix/store` is not a canonical path. More broadly it
// is unclear whether this function should be using the native
@ -94,7 +94,7 @@ StorePath StoreDirConfig::parseStorePath(std::string_view path) const
return StorePath(baseNameOf(p));
}
std::optional<StorePath> StoreDirConfig::maybeParseStorePath(std::string_view path) const
std::optional<StorePath> MixStoreDirMethods::maybeParseStorePath(std::string_view path) const
{
try {
return parseStorePath(path);
@ -103,24 +103,24 @@ std::optional<StorePath> StoreDirConfig::maybeParseStorePath(std::string_view pa
}
}
bool StoreDirConfig::isStorePath(std::string_view path) const
bool MixStoreDirMethods::isStorePath(std::string_view path) const
{
return (bool) maybeParseStorePath(path);
}
StorePathSet StoreDirConfig::parseStorePathSet(const PathSet & paths) const
StorePathSet MixStoreDirMethods::parseStorePathSet(const PathSet & paths) const
{
StorePathSet res;
for (auto & i : paths) res.insert(parseStorePath(i));
return res;
}
std::string StoreDirConfig::printStorePath(const StorePath & path) const
std::string MixStoreDirMethods::printStorePath(const StorePath & path) const
{
return (storeDir + "/").append(path.to_string());
}
PathSet StoreDirConfig::printStorePathSet(const StorePathSet & paths) const
PathSet MixStoreDirMethods::printStorePathSet(const StorePathSet & paths) const
{
PathSet res;
for (auto & i : paths) res.insert(printStorePath(i));

View File

@ -18,17 +18,67 @@
#include "nix/util/callback.hh"
#include "nix/store/filetransfer.hh"
#include "nix/util/signals.hh"
#include "nix/store/config-parse-impl.hh"
#include <nlohmann/json.hpp>
namespace nix {
constexpr static const RemoteStoreConfigT<config::SettingInfo> remoteStoreConfigDescriptions = {
.maxConnections{
.name = "max-connections",
.description = "Maximum number of concurrent connections to the Nix daemon.",
},
.maxConnectionAge{
.name = "max-connection-age",
.description = "Maximum age of a connection before it is closed.",
},
};
#define REMOTE_STORE_CONFIG_FIELDS(X) \
X(maxConnections), \
X(maxConnectionAge),
MAKE_PARSE(RemoteStoreConfig, remoteStoreConfig, REMOTE_STORE_CONFIG_FIELDS)
static RemoteStoreConfigT<config::JustValue> remoteStoreConfigDefaults()
{
return {
.maxConnections = {1},
.maxConnectionAge = {std::numeric_limits<unsigned int>::max()},
};
}
MAKE_APPLY_PARSE(RemoteStoreConfig, remoteStoreConfig, REMOTE_STORE_CONFIG_FIELDS)
config::SettingDescriptionMap RemoteStoreConfig::descriptions()
{
constexpr auto & descriptions = remoteStoreConfigDescriptions;
auto defaults = remoteStoreConfigDefaults();
return {
REMOTE_STORE_CONFIG_FIELDS(DESC_ROW)
};
}
RemoteStore::Config::RemoteStoreConfig(const Store::Config & storeConfig, const StoreReference::Params & params)
: RemoteStoreConfigT<config::JustValue>{remoteStoreConfigApplyParse(params)}
, storeConfig{storeConfig}
{
}
/* TODO: Separate these store types into different files, give them better names */
RemoteStore::RemoteStore(const Params & params)
: RemoteStoreConfig(params)
, Store(params)
RemoteStore::RemoteStore(const Config & config)
: Store{config.storeConfig}
, config{config}
, connections(make_ref<Pool<Connection>>(
std::max(1, maxConnections.get()),
std::max(1, config.maxConnections.get()),
[this]() {
auto conn = openConnectionWrapper();
try {
@ -39,12 +89,12 @@ RemoteStore::RemoteStore(const Params & params)
}
return conn;
},
[this](const ref<Connection> & r) {
[config](const ref<Connection> & r) {
return
r->to.good()
&& r->from.good()
&& std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::steady_clock::now() - r->startTime).count() < maxConnectionAge;
std::chrono::steady_clock::now() - r->startTime).count() < config.maxConnectionAge;
}
))
{
@ -122,7 +172,7 @@ void RemoteStore::setOptions(Connection & conn)
<< settings.useSubstitutes;
if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 12) {
std::map<std::string, Config::SettingInfo> overrides;
std::map<std::string, nix::Config::SettingInfo> overrides;
settings.getSettings(overrides, true); // libstore settings
fileTransferSettings.getSettings(overrides, true);
overrides.erase(settings.keepFailed.name);

View File

@ -30,32 +30,23 @@ bool RestrictionContext::isAllowed(const DerivedPath & req)
return isAllowed(pathPartOfReq(req));
}
struct RestrictedStoreConfig : virtual LocalFSStoreConfig
{
using LocalFSStoreConfig::LocalFSStoreConfig;
const std::string name() override
{
return "Restricted Store";
}
};
/**
* A wrapper around LocalStore that only allows building/querying of
* paths that are in the input closures of the build or were added via
* recursive Nix calls.
*/
struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual IndirectRootStore, public virtual GcStore
struct RestrictedStore : public virtual IndirectRootStore, public virtual GcStore
{
ref<const LocalStore::Config> config;
ref<LocalStore> next;
RestrictionContext & goal;
RestrictedStore(const Params & params, ref<LocalStore> next, RestrictionContext & goal)
: StoreConfig(params)
, LocalFSStoreConfig(params)
, RestrictedStoreConfig(params)
, Store(params)
, LocalFSStore(params)
RestrictedStore(ref<LocalStore::Config> config, ref<LocalStore> next, RestrictionContext & goal)
: Store{*config}
, LocalFSStore{*config}
, config{config}
, next(next)
, goal(goal)
{
@ -63,7 +54,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In
Path getRealStoreDir() override
{
return next->realStoreDir;
return next->config->realStoreDir;
}
std::string getUri() override
@ -176,9 +167,9 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In
}
};
ref<Store> makeRestrictedStore(const Store::Params & params, ref<LocalStore> next, RestrictionContext & context)
ref<Store> makeRestrictedStore(ref<LocalStore::Config> config, ref<LocalStore> next, RestrictionContext & context)
{
return make_ref<RestrictedStore>(params, next, context);
return make_ref<RestrictedStore>(config, next, context);
}
StorePathSet RestrictedStore::queryAllValidPaths()

View File

@ -2,8 +2,6 @@
#if NIX_WITH_S3_SUPPORT
#include <assert.h>
#include "nix/store/s3.hh"
#include "nix/store/nar-info.hh"
#include "nix/store/nar-info-disk-cache.hh"
@ -11,6 +9,8 @@
#include "nix/util/compression.hh"
#include "nix/store/filetransfer.hh"
#include "nix/util/signals.hh"
#include "nix/store/config-parse-impl.hh"
#include "nix/store/store-registration.hh"
#include <aws/core/Aws.h>
#include <aws/core/VersionConfig.h>
@ -235,29 +235,138 @@ S3Helper::FileTransferResult S3Helper::getObject(
return res;
}
S3BinaryCacheStore::S3BinaryCacheStore(const Params & params)
: BinaryCacheStoreConfig(params)
, BinaryCacheStore(params)
{ }
constexpr static const S3BinaryCacheStoreConfigT<config::SettingInfo> s3BinaryCacheStoreConfigDescriptions = {
.profile{
.name = "profile",
.description = R"(
The name of the AWS configuration profile to use. By default
Nix will use the `default` profile.
)",
},
.region{
.name = "region",
.description = R"(
The region of the S3 bucket. If your bucket is not in
`useast-1`, you should always explicitly specify the region
parameter.
)",
},
.scheme{
.name = "scheme",
.description = R"(
The scheme used for S3 requests, `https` (default) or `http`. This
option allows you to disable HTTPS for binary caches which don't
support it.
S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig(
std::string_view uriScheme,
std::string_view bucketName,
const Params & params)
: StoreConfig(params)
, BinaryCacheStoreConfig(params)
, bucketName(bucketName)
> **Note**
>
> HTTPS should be used if the cache might contain sensitive
> information.
)",
},
.endpoint{
.name = "endpoint",
.description = R"(
The URL of the endpoint of an S3-compatible service such as MinIO.
Do not specify this setting if you're using Amazon S3.
> **Note**
>
> This endpoint must support HTTPS and will use path-based
> addressing instead of virtual host based addressing.
)",
},
.narinfoCompression{
.name = "narinfo-compression",
.description = "Compression method for `.narinfo` files.",
},
.lsCompression{
.name = "ls-compression",
.description = "Compression method for `.ls` files.",
},
.logCompression{
.name = "log-compression",
.description = R"(
Compression method for `log/*` files. It is recommended to
use a compression method supported by most web browsers
(e.g. `brotli`).
)",
},
.multipartUpload{
.name = "multipart-upload",
.description = "Whether to use multi-part uploads.",
},
.bufferSize{
.name = "buffer-size",
.description = "Size (in bytes) of each part in multi-part uploads.",
},
};
#define S3_BINARY_CACHE_STORE_CONFIG_FIELDS(X) \
X(profile), \
X(region), \
X(scheme), \
X(endpoint), \
X(narinfoCompression), \
X(lsCompression), \
X(logCompression), \
X(multipartUpload), \
X(bufferSize),
MAKE_PARSE(S3BinaryCacheStoreConfig, s3BinaryCacheStoreConfig, S3_BINARY_CACHE_STORE_CONFIG_FIELDS)
static S3BinaryCacheStoreConfigT<config::JustValue> s3BinaryCacheStoreConfigDefaults()
{
// Don't want to use use AWS SDK in header, so we check the default
// here. TODO do this better after we overhaul the store settings
// system.
assert(std::string{defaultRegion} == std::string{Aws::Region::US_EAST_1});
if (bucketName.empty())
throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme);
return {
.profile = {""},
.region = {Aws::Region::US_EAST_1},
.scheme = {""},
.endpoint = {""},
.narinfoCompression = {""},
.lsCompression = {""},
.logCompression = {""},
.multipartUpload = {false},
.bufferSize = {5 * 1024 * 1024},
};
}
MAKE_APPLY_PARSE(S3BinaryCacheStoreConfig, s3BinaryCacheStoreConfig, S3_BINARY_CACHE_STORE_CONFIG_FIELDS)
config::SettingDescriptionMap S3BinaryCacheStoreConfig::descriptions()
{
config::SettingDescriptionMap ret;
ret.merge(StoreConfig::descriptions());
ret.merge(BinaryCacheStoreConfig::descriptions());
{
constexpr auto & descriptions = s3BinaryCacheStoreConfigDescriptions;
auto defaults = s3BinaryCacheStoreConfigDefaults();
ret.merge(decltype(ret){
S3_BINARY_CACHE_STORE_CONFIG_FIELDS(DESC_ROW)
});
}
return ret;
}
S3BinaryCacheStore::Config::S3BinaryCacheStoreConfig(
std::string_view scheme,
std::string_view authority,
const StoreReference::Params & params)
: Store::Config{params}
, BinaryCacheStore::Config{*this, params}
, S3BinaryCacheStoreConfigT<config::JustValue>{s3BinaryCacheStoreConfigApplyParse(params)}
, bucketName{authority}
{
if (bucketName.empty())
throw UsageError("`%s` store requires a bucket name in its Store URI", scheme);
}
S3BinaryCacheStore::S3BinaryCacheStore(ref<const Config> config)
: BinaryCacheStore(*config)
, config{config}
{ }
std::string S3BinaryCacheStoreConfig::doc()
{
return
@ -266,40 +375,39 @@ std::string S3BinaryCacheStoreConfig::doc()
}
struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore
struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore
{
Stats stats;
S3Helper s3Helper;
S3BinaryCacheStoreImpl(
std::string_view uriScheme,
std::string_view bucketName,
const Params & params)
: StoreConfig(params)
, BinaryCacheStoreConfig(params)
, S3BinaryCacheStoreConfig(uriScheme, bucketName, params)
, Store(params)
, BinaryCacheStore(params)
, S3BinaryCacheStore(params)
, s3Helper(profile, region, scheme, endpoint)
S3BinaryCacheStoreImpl(ref<const Config> config)
: Store{*config}
, BinaryCacheStore{*config}
, S3BinaryCacheStore{config}
, s3Helper(config->profile, config->region, config->scheme, config->endpoint)
{
diskCache = getNarInfoDiskCache();
init();
}
std::string getUri() override
{
return "s3://" + bucketName;
return "s3://" + config->bucketName;
}
void init() override
{
if (auto cacheInfo = diskCache->upToDateCacheExists(getUri())) {
wantMassQuery.setDefault(cacheInfo->wantMassQuery);
priority.setDefault(cacheInfo->priority);
resolvedSubstConfig.wantMassQuery.value =
config->storeConfig.wantMassQuery.optValue.value_or(cacheInfo->wantMassQuery);
resolvedSubstConfig.priority.value =
config->storeConfig.priority.optValue.value_or(cacheInfo->priority);
} else {
BinaryCacheStore::init();
diskCache->createCache(getUri(), storeDir, wantMassQuery, priority);
diskCache->createCache(
getUri(), config->storeDir, resolvedSubstConfig.wantMassQuery, resolvedSubstConfig.priority);
}
}
@ -328,7 +436,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
auto res = s3Helper.client->HeadObject(
Aws::S3::Model::HeadObjectRequest()
.WithBucket(bucketName)
.WithBucket(config->bucketName)
.WithKey(path));
if (!res.IsSuccess()) {
@ -372,7 +480,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
const std::string & mimeType,
const std::string & contentEncoding)
{
std::string uri = "s3://" + bucketName + "/" + path;
std::string uri = "s3://" + config->bucketName + "/" + path;
Activity act(*logger, lvlTalkative, actFileTransfer,
fmt("uploading '%s'", uri),
Logger::Fields{uri}, getCurActivity());
@ -387,11 +495,11 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
std::call_once(transferManagerCreated, [&]()
{
if (multipartUpload) {
if (config->multipartUpload) {
TransferManagerConfiguration transferConfig(executor.get());
transferConfig.s3Client = s3Helper.client;
transferConfig.bufferSize = bufferSize;
transferConfig.bufferSize = config->bufferSize;
transferConfig.uploadProgressCallback =
[](const TransferManager * transferManager,
@ -421,6 +529,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
auto now1 = std::chrono::steady_clock::now();
auto & bucketName = config->bucketName;
if (transferManager) {
if (contentEncoding != "")
@ -508,12 +618,12 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
return std::make_shared<std::stringstream>(std::move(compressed));
};
if (narinfoCompression != "" && hasSuffix(path, ".narinfo"))
uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression);
else if (lsCompression != "" && hasSuffix(path, ".ls"))
uploadFile(path, compress(lsCompression), mimeType, lsCompression);
else if (logCompression != "" && hasPrefix(path, "log/"))
uploadFile(path, compress(logCompression), mimeType, logCompression);
if (config->narinfoCompression != "" && hasSuffix(path, ".narinfo"))
uploadFile(path, compress(config->narinfoCompression), mimeType, config->narinfoCompression);
else if (config->lsCompression != "" && hasSuffix(path, ".ls"))
uploadFile(path, compress(config->lsCompression), mimeType, config->lsCompression);
else if (config->logCompression != "" && hasPrefix(path, "log/"))
uploadFile(path, compress(config->logCompression), mimeType, config->logCompression);
else
uploadFile(path, istream, mimeType, "");
}
@ -523,14 +633,14 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
stats.get++;
// FIXME: stream output to sink.
auto res = s3Helper.getObject(bucketName, path);
auto res = s3Helper.getObject(config->bucketName, path);
stats.getBytes += res.data ? res.data->size() : 0;
stats.getTimeMs += res.durationMs;
if (res.data) {
printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms",
bucketName, path, res.data->size(), res.durationMs);
config->bucketName, path, res.data->size(), res.durationMs);
sink(*res.data);
} else
@ -542,6 +652,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
StorePathSet paths;
std::string marker;
auto & bucketName = config->bucketName;
do {
debug("listing bucket 's3://%s' from key '%s'...", bucketName, marker);
@ -580,7 +692,12 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
}
};
static RegisterStoreImplementation<S3BinaryCacheStoreImpl, S3BinaryCacheStoreConfig> regS3BinaryCacheStore;
ref<Store> S3BinaryCacheStoreImpl::Config::openStore() const
{
return make_ref<S3BinaryCacheStoreImpl>(ref{shared_from_this()});
}
static RegisterStoreImplementation<S3BinaryCacheStoreImpl::Config> regS3BinaryCacheStore;
}

View File

@ -1,3 +1,4 @@
#include "nix/util/json-utils.hh"
#include "nix/store/ssh-store.hh"
#include "nix/store/local-fs-store.hh"
#include "nix/store/remote-store-connection.hh"
@ -7,19 +8,105 @@
#include "nix/store/worker-protocol-impl.hh"
#include "nix/util/pool.hh"
#include "nix/store/ssh.hh"
#include "nix/store/config-parse-impl.hh"
#include "nix/store/store-registration.hh"
namespace nix {
constexpr static const SSHStoreConfigT<config::SettingInfo> sshStoreConfigDescriptions = {
.remoteProgram{
.name = "remote-program",
.description = "Path to the `nix-daemon` executable on the remote machine.",
},
};
#define SSH_STORE_CONFIG_FIELDS(X) \
X(remoteProgram)
MAKE_PARSE(SSHStoreConfig, sshStoreConfig, SSH_STORE_CONFIG_FIELDS)
static SSHStoreConfigT<config::JustValue> sshStoreConfigDefaults()
{
return {
.remoteProgram = {{"nix-daemon"}},
};
}
MAKE_APPLY_PARSE(SSHStoreConfig, sshStoreConfig, SSH_STORE_CONFIG_FIELDS)
config::SettingDescriptionMap SSHStoreConfig::descriptions()
{
config::SettingDescriptionMap ret;
ret.merge(StoreConfig::descriptions());
ret.merge(CommonSSHStoreConfig::descriptions());
ret.merge(RemoteStoreConfig::descriptions());
{
constexpr auto & descriptions = sshStoreConfigDescriptions;
auto defaults = sshStoreConfigDefaults();
ret.merge(decltype(ret){
SSH_STORE_CONFIG_FIELDS(DESC_ROW)
});
}
ret.insert_or_assign(
"mounted",
config::SettingDescription{
.description = stripIndentation(R"(
If this nested settings object is defined (`{..}` not `null`), additionally requires that store be mounted in the local file system.
The mounting of that store is not managed by Nix, and must by managed manually.
It could be accomplished with SSHFS or NFS, for example.
The local file system is used to optimize certain operations.
For example, rather than serializing Nix archives and sending over the Nix channel,
we can directly access the file system data via the mount-point.
The local file system is also used to make certain operations possible that wouldn't otherwise be.
For example, persistent GC roots can be created if they reside on the same file system as the remote store:
the remote side will create the symlinks necessary to avoid race conditions.
)"),
.experimentalFeature = Xp::MountedSSHStore,
.info = config::SettingDescription::Sub{
.nullable = true,
.map = LocalFSStoreConfig::descriptions()
},
});
return ret;
}
static std::optional<LocalFSStore::Config> getMounted(
const Store::Config & storeConfig,
const StoreReference::Params & params,
const ExperimentalFeatureSettings & xpSettings)
{
auto mountedParamsOpt = optionalValueAt(params, "mounted");
if (!mountedParamsOpt) return {};
auto * mountedParamsP = getNullable(*mountedParamsOpt);
xpSettings.require(Xp::MountedSSHStore);
if (!mountedParamsP) return {};
auto & mountedParams = getObject(*mountedParamsP);
return {{storeConfig, mountedParams}};
}
SSHStoreConfig::SSHStoreConfig(
std::string_view scheme,
std::string_view authority,
const Params & params)
: StoreConfig(params)
, RemoteStoreConfig(params)
, CommonSSHStoreConfig(scheme, authority, params)
const StoreReference::Params & params, const ExperimentalFeatureSettings & xpSettings)
: Store::Config{params}
, RemoteStore::Config{*this, params}
, CommonSSHStoreConfig{scheme, authority, params}
, SSHStoreConfigT<config::JustValue>{sshStoreConfigApplyParse(params)}
, mounted{getMounted(*this, params, xpSettings)}
{
}
std::string SSHStoreConfig::doc()
{
return
@ -27,21 +114,18 @@ std::string SSHStoreConfig::doc()
;
}
class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore
{
public:
SSHStore(
std::string_view scheme,
std::string_view host,
const Params & params)
: StoreConfig(params)
, RemoteStoreConfig(params)
, CommonSSHStoreConfig(scheme, host, params)
, SSHStoreConfig(scheme, host, params)
, Store(params)
, RemoteStore(params)
, master(createSSHMaster(
struct SSHStore : virtual RemoteStore
{
using Config = SSHStoreConfig;
ref<const Config> config;
SSHStore(ref<const Config> config)
: Store{*config}
, RemoteStore{*config}
, config{config}
, master(config->createSSHMaster(
// Use SSH master only if using more than 1 connection.
connections->capacity() > 1))
{
@ -49,7 +133,7 @@ public:
std::string getUri() override
{
return *uriSchemes().begin() + "://" + host;
return *Config::uriSchemes().begin() + "://" + host;
}
// FIXME extend daemon protocol, move implementation to RemoteStore
@ -88,32 +172,6 @@ protected:
};
MountedSSHStoreConfig::MountedSSHStoreConfig(StringMap params)
: StoreConfig(params)
, RemoteStoreConfig(params)
, CommonSSHStoreConfig(params)
, SSHStoreConfig(params)
, LocalFSStoreConfig(params)
{
}
MountedSSHStoreConfig::MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params)
: StoreConfig(params)
, RemoteStoreConfig(params)
, CommonSSHStoreConfig(scheme, host, params)
, SSHStoreConfig(params)
, LocalFSStoreConfig(params)
{
}
std::string MountedSSHStoreConfig::doc()
{
return
#include "mounted-ssh-store.md"
;
}
/**
* The mounted ssh store assumes that filesystems on the remote host are
* shared with the local host. This means that the remote nix store is
@ -128,35 +186,24 @@ std::string MountedSSHStoreConfig::doc()
* The difference lies in how they manage GC roots. See addPermRoot
* below for details.
*/
class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSHStore, public virtual LocalFSStore
struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore
{
public:
using Config = SSHStore::Config;
MountedSSHStore(
std::string_view scheme,
std::string_view host,
const Params & params)
: StoreConfig(params)
, RemoteStoreConfig(params)
, CommonSSHStoreConfig(scheme, host, params)
, SSHStoreConfig(params)
, LocalFSStoreConfig(params)
, MountedSSHStoreConfig(params)
, Store(params)
, RemoteStore(params)
, SSHStore(scheme, host, params)
, LocalFSStore(params)
const LocalFSStore::Config & mountedConfig;
MountedSSHStore(ref<const Config> config, const LocalFSStore::Config & mountedConfig)
: Store{*config}
, RemoteStore{*config}
, SSHStore{config}
, LocalFSStore{mountedConfig}
, mountedConfig{mountedConfig}
{
extraRemoteProgramArgs = {
"--process-ops",
};
}
std::string getUri() override
{
return *uriSchemes().begin() + "://" + host;
}
void narFromPath(const StorePath & path, Sink & sink) override
{
return LocalFSStore::narFromPath(path, sink);
@ -198,14 +245,25 @@ public:
}
};
ref<Store> MountedSSHStore::Config::openStore() const {
ref config {shared_from_this()};
if (config->mounted)
return make_ref<MountedSSHStore>(config, *config->mounted);
else
return make_ref<SSHStore>(config);
}
ref<RemoteStore::Connection> SSHStore::openConnection()
{
auto conn = make_ref<Connection>();
Strings command = remoteProgram.get();
Strings command = config->remoteProgram.get();
command.push_back("--stdio");
if (remoteStore.get() != "") {
if (config->remoteStore.get() != "") {
command.push_back("--store");
command.push_back(remoteStore.get());
command.push_back(config->remoteStore.get());
}
command.insert(command.end(),
extraRemoteProgramArgs.begin(), extraRemoteProgramArgs.end());
@ -215,7 +273,7 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
return conn;
}
static RegisterStoreImplementation<SSHStore, SSHStoreConfig> regSSHStore;
static RegisterStoreImplementation<MountedSSHStore, MountedSSHStoreConfig> regMountedSSHStore;
static RegisterStoreImplementation<SSHStore::Config> regSSHStore;
static RegisterStoreImplementation<MountedSSHStore::Config> regMountedSSHStore;
}

View File

@ -4,5 +4,4 @@ R"(
Experimental store type that allows full access to a Nix store on a
remote machine.
)"

View File

@ -5,6 +5,7 @@
#include "nix/store/realisation.hh"
#include "nix/store/derivations.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/util/util.hh"
#include "nix/store/nar-info-disk-cache.hh"
#include "nix/util/thread-pool.hh"
@ -18,6 +19,7 @@
#include "nix/store/worker-protocol.hh"
#include "nix/util/signals.hh"
#include "nix/util/users.hh"
#include "nix/store/config-parse-impl.hh"
#include <filesystem>
#include <nlohmann/json.hpp>
@ -28,14 +30,13 @@ using json = nlohmann::json;
namespace nix {
bool StoreDirConfig::isInStore(PathView path) const
bool MixStoreDirMethods::isInStore(PathView path) const
{
return isInDir(path, storeDir);
}
std::pair<StorePath, Path> StoreDirConfig::toStorePath(PathView path) const
std::pair<StorePath, Path> MixStoreDirMethods::toStorePath(PathView path) const
{
if (!isInStore(path))
throw Error("path '%1%' is not in the Nix store", path);
@ -77,7 +78,7 @@ to match.
*/
StorePath StoreDirConfig::makeStorePath(std::string_view type,
StorePath MixStoreDirMethods::makeStorePath(std::string_view type,
std::string_view hash, std::string_view name) const
{
/* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
@ -88,14 +89,14 @@ StorePath StoreDirConfig::makeStorePath(std::string_view type,
}
StorePath StoreDirConfig::makeStorePath(std::string_view type,
StorePath MixStoreDirMethods::makeStorePath(std::string_view type,
const Hash & hash, std::string_view name) const
{
return makeStorePath(type, hash.to_string(HashFormat::Base16, true), name);
}
StorePath StoreDirConfig::makeOutputPath(std::string_view id,
StorePath MixStoreDirMethods::makeOutputPath(std::string_view id,
const Hash & hash, std::string_view name) const
{
return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id));
@ -106,7 +107,7 @@ StorePath StoreDirConfig::makeOutputPath(std::string_view id,
hacky, but we can't put them in, say, <s2> (per the grammar above)
since that would be ambiguous. */
static std::string makeType(
const StoreDirConfig & store,
const MixStoreDirMethods & store,
std::string && type,
const StoreReferences & references)
{
@ -119,7 +120,7 @@ static std::string makeType(
}
StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
StorePath MixStoreDirMethods::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
{
if (info.method == FileIngestionMethod::Git && info.hash.algo != HashAlgorithm::SHA1)
throw Error("Git file ingestion must use SHA-1 hash");
@ -141,7 +142,7 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed
}
StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const
StorePath MixStoreDirMethods::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const
{
// New template
return std::visit(overloaded {
@ -162,7 +163,7 @@ StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const
}
std::pair<StorePath, Hash> StoreDirConfig::computeStorePath(
std::pair<StorePath, Hash> MixStoreDirMethods::computeStorePath(
std::string_view name,
const SourcePath & path,
ContentAddressMethod method,
@ -188,6 +189,114 @@ std::pair<StorePath, Hash> StoreDirConfig::computeStorePath(
}
constexpr static const StoreConfigT<config::SettingInfo> storeConfigDescriptions = {
.pathInfoCacheSize{
.name = "path-info-cache-size",
.description = "Size of the in-memory store path metadata cache.",
},
.isTrusted{
.name = "trusted",
.description = R"(
Whether paths from this store can be used as substitutes
even if they are not signed by a key listed in the
[`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys)
setting.
)",
},
.systemFeatures{
.name = "system-features",
.description = R"(
Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations.
Example: `"kvm"`
)",
// The default value is CPU- and OS-specific, and thus
// unsuitable to be rendered in the documentation.
.documentDefault = false,
},
};
constexpr static const SubstituterConfigT<config::SettingInfo> substituterConfigDescriptions = {
.priority{
.name = "priority",
.description = R"(
Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters).
A lower value means a higher priority.
)",
},
.wantMassQuery{
.name = "want-mass-query",
.description = R"(
Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters).
)",
},
};
#define STORE_CONFIG_FIELDS(X) \
X(pathInfoCacheSize), \
X(isTrusted), \
X(systemFeatures),
#define SUBSTITUTER_CONFIG_FIELDS(X) \
X(priority), \
X(wantMassQuery),
MAKE_PARSE(StoreConfig, storeConfig, STORE_CONFIG_FIELDS)
MAKE_PARSE(SubstituterConfig, substituterConfig, SUBSTITUTER_CONFIG_FIELDS)
static StoreConfigT<config::JustValue> storeConfigDefaults()
{
return {
.pathInfoCacheSize = {65536},
.isTrusted = {false},
.systemFeatures = {StoreConfig::getDefaultSystemFeatures()},
};
};
SubstituterConfigT<config::JustValue> substituterConfigDefaults()
{
return {
.priority = {0},
.wantMassQuery = {false},
};
};
MAKE_APPLY_PARSE(StoreConfig, storeConfig, STORE_CONFIG_FIELDS)
Store::Config::StoreConfig(const StoreReference::Params & params)
: StoreDirConfig{params}
, StoreConfigT<config::JustValue>{storeConfigApplyParse(params)}
, SubstituterConfigT<config::OptValue>{substituterConfigParse(params)}
{
}
config::SettingDescriptionMap StoreConfig::descriptions()
{
auto ret = StoreDirConfig::descriptions();
{
constexpr auto & descriptions = storeConfigDescriptions;
auto defaults = storeConfigDefaults();
ret.merge(config::SettingDescriptionMap {
STORE_CONFIG_FIELDS(DESC_ROW)
});
}
{
constexpr auto & descriptions = substituterConfigDescriptions;
auto defaults = substituterConfigDefaults();
ret.merge(config::SettingDescriptionMap {
SUBSTITUTER_CONFIG_FIELDS(DESC_ROW)
});
};
return ret;
}
StorePath Store::addToStore(
std::string_view name,
const SourcePath & path,
@ -420,7 +529,7 @@ ValidPathInfo Store::addToStoreSlow(
return info;
}
StringSet StoreConfig::getDefaultSystemFeatures()
StringSet Store::Config::getDefaultSystemFeatures()
{
auto res = settings.systemFeatures.get();
@ -433,9 +542,10 @@ StringSet StoreConfig::getDefaultSystemFeatures()
return res;
}
Store::Store(const Params & params)
: StoreConfig(params)
, state({(size_t) pathInfoCacheSize})
Store::Store(const Store::Config & config)
: MixStoreDirMethods{config}
, config{config}
, state({(size_t) config.pathInfoCacheSize})
{
assertLibStoreInitialized();
}
@ -1205,7 +1315,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre
}
std::string StoreDirConfig::showPaths(const StorePathSet & paths)
std::string MixStoreDirMethods::showPaths(const StorePathSet & paths) const
{
std::string s;
for (auto & i : paths) {
@ -1303,102 +1413,3 @@ void Store::signRealisation(Realisation & realisation)
}
}
#include "nix/store/local-store.hh"
#include "nix/store/uds-remote-store.hh"
namespace nix {
ref<Store> openStore(const std::string & uri,
const Store::Params & extraParams)
{
return openStore(StoreReference::parse(uri, extraParams));
}
ref<Store> openStore(StoreReference && storeURI)
{
auto & params = storeURI.params;
auto store = std::visit(overloaded {
[&](const StoreReference::Auto &) -> std::shared_ptr<Store> {
auto stateDir = getOr(params, "state", settings.nixStateDir);
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
return std::make_shared<LocalStore>(params);
else if (pathExists(settings.nixDaemonSocketFile))
return std::make_shared<UDSRemoteStore>(params);
#ifdef __linux__
else if (!pathExists(stateDir)
&& params.empty()
&& !isRootUser()
&& !getEnv("NIX_STORE_DIR").has_value()
&& !getEnv("NIX_STATE_DIR").has_value())
{
/* If /nix doesn't exist, there is no daemon socket, and
we're not root, then automatically set up a chroot
store in ~/.local/share/nix/root. */
auto chrootStore = getDataDir() + "/root";
if (!pathExists(chrootStore)) {
try {
createDirs(chrootStore);
} catch (SystemError & e) {
return std::make_shared<LocalStore>(params);
}
warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
} else
debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
return std::make_shared<LocalStore>("local", chrootStore, params);
}
#endif
else
return std::make_shared<LocalStore>(params);
},
[&](const StoreReference::Specified & g) {
for (const auto & implem : *Implementations::registered)
if (implem.uriSchemes.count(g.scheme))
return implem.create(g.scheme, g.authority, params);
throw Error("don't know how to open Nix store with scheme '%s'", g.scheme);
},
}, storeURI.variant);
experimentalFeatureSettings.require(store->experimentalFeature());
store->warnUnknownSettings();
store->init();
return ref<Store> { store };
}
std::list<ref<Store>> getDefaultSubstituters()
{
static auto stores([]() {
std::list<ref<Store>> stores;
StringSet done;
auto addStore = [&](const std::string & uri) {
if (!done.insert(uri).second) return;
try {
stores.push_back(openStore(uri));
} catch (Error & e) {
logWarning(e.info());
}
};
for (const auto & uri : settings.substituters.get())
addStore(uri);
stores.sort([](ref<Store> & a, ref<Store> & b) {
return a->priority < b->priority;
});
return stores;
} ());
return stores;
}
std::vector<StoreFactory> * Implementations::registered = 0;
}

View File

@ -0,0 +1,45 @@
#include "nix/store/store-dir-config.hh"
#include "nix/store/config-parse-impl.hh"
#include "nix/util/util.hh"
#include "nix/store/globals.hh"
namespace nix {
constexpr static const StoreDirConfigT<config::SettingInfo> storeDirConfigDescriptions = {
._storeDir{
.name = "store",
.description = R"(
Logical location of the Nix store, usually
`/nix/store`. Note that you can only copy store paths
between stores if they have the same `store` setting.
)",
},
};
#define STORE_DIR_CONFIG_FIELDS(X) X(_storeDir),
MAKE_PARSE(StoreDirConfig, storeDirConfig, STORE_DIR_CONFIG_FIELDS)
static StoreDirConfigT<config::JustValue> storeDirConfigDefaults()
{
return {
._storeDir = {settings.nixStore},
};
}
MAKE_APPLY_PARSE(StoreDirConfig, storeDirConfig, STORE_DIR_CONFIG_FIELDS)
StoreDirConfig::StoreDirConfig(const StoreReference::Params & params)
: StoreDirConfigT<config::JustValue>{storeDirConfigApplyParse(params)}
, MixStoreDirMethods{_storeDir.value}
{
}
config::SettingDescriptionMap StoreDirConfig::descriptions()
{
constexpr auto & descriptions = storeDirConfigDescriptions;
auto defaults = storeDirConfigDefaults();
return {STORE_DIR_CONFIG_FIELDS(DESC_ROW)};
}
}

View File

@ -1,13 +1,18 @@
#include <regex>
#include <nlohmann/json.hpp>
#include "nix/util/error.hh"
#include "nix/util/url.hh"
#include "nix/store/store-reference.hh"
#include "nix/util/file-system.hh"
#include "nix/util/util.hh"
#include "nix/util/json-utils.hh"
namespace nix {
bool StoreReference::operator==(const StoreReference & rhs) const = default;
static bool isNonUriPath(const std::string & spec)
{
return
@ -33,20 +38,96 @@ std::string StoreReference::render() const
},
variant);
StringMap params2;
for (auto & [k, v] : params) {
auto * p = v.get_ptr<const nlohmann::json::string_t *>();
// if it is a JSON string, just use that
// FIXME: Ensure the literal string isn't itself valid JSON. If
// it is, we still need to dump to escape it.
params2.insert_or_assign(k, p ? *p : v.dump());
}
if (!params.empty()) {
res += "?";
res += encodeQuery(params);
res += encodeQuery(params2);
}
return res;
}
static StoreReference::Params decodeParamsJson(StringMap paramsRaw)
{
StoreReference::Params params;
for (auto && [k, v] : std::move(paramsRaw)) {
nlohmann::json j;
/* We have to parse the URL un an "untyped" way before we do a
"typed" conversion to specific store-configuration types. As
such, the best we can do for back-compat is just white-list
specific query parameter names.
These are all the boolean store parameters in use at the time
of the introduction of JSON store configuration, as evidenced
by `git grep 'F<bool>'`. So these will continue working with
"yes"/"no"/"1"/"0", whereas any new ones will require
"true"/"false".
*/
bool preJsonBool =
std::set<std::string_view>{
"check-mount",
"compress",
"trusted",
"multipart-upload",
"parallel-compression",
"read-only",
"require-sigs",
"want-mass-query",
"index-debug-info",
"write-nar-listing",
}
.contains(std::string_view{k});
auto warnPreJson = [&] {
warn(
"in query param '%s', using '%s' to mean a boolean is deprecated, please use valid JSON 'true' or 'false'",
k,
v);
};
if (preJsonBool && (v == "yes" || v == "1")) {
j = true;
warnPreJson();
} else if (preJsonBool && (v == "no" || v == "0")) {
j = true;
warnPreJson();
} else {
try {
j = nlohmann::json::parse(v);
} catch (nlohmann::json::exception &) {
// if its not valid JSON...
if (k == "remote-program" || k == "system-features") {
// Back compat hack! Split and take that array
j = tokenizeString<std::vector<std::string>>(v);
} else {
// ...keep the literal string.
j = std::move(v);
}
}
}
params.insert_or_assign(std::move(k), std::move(j));
}
return params;
}
StoreReference StoreReference::parse(const std::string & uri, const StoreReference::Params & extraParams)
{
auto params = extraParams;
try {
auto parsedUri = parseURL(uri);
params.insert(parsedUri.query.begin(), parsedUri.query.end());
{
auto params2 = decodeParamsJson(std::move(parsedUri.query));
params.insert(params2.begin(), params2.end());
}
auto baseURI = parsedUri.authority.value_or("") + parsedUri.path;
@ -104,13 +185,75 @@ StoreReference StoreReference::parse(const std::string & uri, const StoreReferen
std::pair<std::string, StoreReference::Params> splitUriAndParams(const std::string & uri_)
{
auto uri(uri_);
StoreReference::Params params;
StringMap params;
auto q = uri.find('?');
if (q != std::string::npos) {
params = decodeQuery(uri.substr(q + 1));
uri = uri_.substr(0, q);
}
return {uri, params};
return {uri, decodeParamsJson(std::move(params))};
}
}
namespace nlohmann {
StoreReference adl_serializer<StoreReference>::from_json(const json & json)
{
StoreReference ref;
switch (json.type()) {
case json::value_t::string: {
ref = StoreReference::parse(json.get_ref<const std::string &>());
break;
}
case json::value_t::object: {
auto & obj = getObject(json);
auto scheme = getString(valueAt(obj, "scheme"));
auto variant = scheme == "auto" ? (StoreReference::Variant{StoreReference::Auto{}})
: (StoreReference::Variant{StoreReference::Specified{
.scheme = scheme,
.authority = getString(valueAt(obj, "authority")),
}});
auto params = obj;
params.erase("scheme");
params.erase("authority");
ref = StoreReference{
.variant = std::move(variant),
.params = std::move(params),
};
break;
}
case json::value_t::null:
case json::value_t::number_unsigned:
case json::value_t::number_integer:
case json::value_t::number_float:
case json::value_t::boolean:
case json::value_t::array:
case json::value_t::binary:
case json::value_t::discarded:
default:
throw UsageError(
"Invalid JSON for Store configuration: is type '%s' but must be string or object", json.type_name());
};
return ref;
}
void adl_serializer<StoreReference>::to_json(json & obj, StoreReference s)
{
obj = s.params;
std::visit(
overloaded{
[&](const StoreReference::Auto &) { obj.emplace("scheme", "auto"); },
[&](const StoreReference::Specified & g) {
obj.emplace("scheme", g.scheme);
obj.emplace("authority", g.authority);
},
},
s.variant);
}
}

View File

@ -0,0 +1,118 @@
#include "nix/store/store-registration.hh"
#include "nix/store/store-open.hh"
#include "nix/store/local-store.hh"
#include "nix/store/uds-remote-store.hh"
#include "nix/util/json-utils.hh"
namespace nix {
ref<Store> openStore(const std::string & uri, const StoreReference::Params & extraParams)
{
return openStore(StoreReference::parse(uri, extraParams));
}
ref<Store> openStore(StoreReference && storeURI)
{
return resolveStoreConfig(std::move(storeURI))->openStore();
}
ref<StoreConfig> resolveStoreConfig(StoreReference && storeURI)
{
auto & params = storeURI.params;
auto storeConfig = std::visit(
overloaded{
[&](const StoreReference::Auto &) -> ref<StoreConfig> {
auto stateDir = getString(getOr(params, "state", settings.nixStateDir));
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
return make_ref<LocalStore::Config>(params);
else if (pathExists(settings.nixDaemonSocketFile))
return make_ref<UDSRemoteStore::Config>(params);
#ifdef __linux__
else if (
!pathExists(stateDir) && params.empty() && !isRootUser() && !getEnv("NIX_STORE_DIR").has_value()
&& !getEnv("NIX_STATE_DIR").has_value()) {
/* If /nix doesn't exist, there is no daemon socket, and
we're not root, then automatically set up a chroot
store in ~/.local/share/nix/root. */
auto chrootStore = getDataDir() + "/root";
if (!pathExists(chrootStore)) {
try {
createDirs(chrootStore);
} catch (SystemError & e) {
return make_ref<LocalStore::Config>(params);
}
warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
} else
debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
return make_ref<LocalStore::Config>("local", chrootStore, params);
}
#endif
else
return make_ref<LocalStore::Config>(params);
},
[&](const StoreReference::Specified & g) {
for (auto & [name, implem] : *Implementations::registered)
if (implem.uriSchemes.count(g.scheme))
return implem.parseConfig(g.scheme, g.authority, params);
throw Error("don't know how to open Nix store with scheme '%s'", g.scheme);
},
},
storeURI.variant);
experimentalFeatureSettings.require(storeConfig->experimentalFeature());
return storeConfig;
}
Implementations::V * Implementations::registered = 0;
std::list<ref<Store>> getDefaultSubstituters()
{
static auto stores([]() {
std::list<ref<Store>> stores;
StringSet done;
auto addStore = [&](const std::string & uri) {
if (!done.insert(uri).second)
return;
try {
stores.push_back(openStore(uri));
} catch (Error & e) {
logWarning(e.info());
}
};
for (auto uri : settings.substituters.get())
addStore(uri);
stores.sort([](ref<Store> & a, ref<Store> & b) {
return a->resolvedSubstConfig.priority < b->resolvedSubstConfig.priority;
});
return stores;
}());
return stores;
}
}
namespace nlohmann {
using namespace nix::config;
ref<const StoreConfig> adl_serializer<ref<const StoreConfig>>::from_json(const json & json)
{
return resolveStoreConfig(adl_serializer<StoreReference>::from_json(json));
}
void adl_serializer<ref<const StoreConfig>>::to_json(json & obj, ref<const StoreConfig> s)
{
// TODO, for tests maybe
assert(false);
}
}

View File

@ -1,6 +1,7 @@
#include "nix/store/uds-remote-store.hh"
#include "nix/util/unix-domain-socket.hh"
#include "nix/store/worker-protocol.hh"
#include "nix/store/store-registration.hh"
#include <sys/types.h>
#include <sys/stat.h>
@ -17,16 +18,26 @@
namespace nix {
config::SettingDescriptionMap UDSRemoteStoreConfig::descriptions()
{
config::SettingDescriptionMap ret;
ret.merge(StoreConfig::descriptions());
ret.merge(LocalFSStoreConfig::descriptions());
ret.merge(RemoteStoreConfig::descriptions());
return ret;
}
UDSRemoteStoreConfig::UDSRemoteStoreConfig(
std::string_view scheme,
std::string_view authority,
const Params & params)
: StoreConfig(params)
, LocalFSStoreConfig(params)
, RemoteStoreConfig(params)
const StoreReference::Params & params)
: Store::Config{params}
, LocalFSStore::Config{*this, params}
, RemoteStore::Config{*this, params}
, path{authority.empty() ? settings.nixDaemonSocketFile : authority}
{
if (scheme != UDSRemoteStoreConfig::scheme) {
if (uriSchemes().count(std::string{scheme}) == 0) {
throw UsageError("Scheme must be 'unix'");
}
}
@ -40,36 +51,24 @@ std::string UDSRemoteStoreConfig::doc()
}
// A bit gross that we now pass empty string but this is knowing that
// empty string will later default to the same nixDaemonSocketFile. Why
// don't we just wire it all through? I believe there are cases where it
// will live reload so we want to continue to account for that.
UDSRemoteStore::UDSRemoteStore(const Params & params)
: UDSRemoteStore(scheme, "", params)
{}
UDSRemoteStore::UDSRemoteStore(std::string_view scheme, std::string_view authority, const Params & params)
: StoreConfig(params)
, LocalFSStoreConfig(params)
, RemoteStoreConfig(params)
, UDSRemoteStoreConfig(scheme, authority, params)
, Store(params)
, LocalFSStore(params)
, RemoteStore(params)
UDSRemoteStore::UDSRemoteStore(ref<const Config> config)
: Store{*config}
, LocalFSStore{*config}
, RemoteStore{*config}
, config{config}
{
}
std::string UDSRemoteStore::getUri()
{
return path == settings.nixDaemonSocketFile
return config->path == settings.nixDaemonSocketFile
? // FIXME: Not clear why we return daemon here and not default
// to settings.nixDaemonSocketFile
//
// unix:// with no path also works. Change what we return?
"daemon"
: std::string(scheme) + "://" + path;
: std::string(*Config::uriSchemes().begin()) + "://" + config->path;
}
@ -86,7 +85,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
/* Connect to a daemon that does the privileged work for us. */
conn->fd = createUnixDomainSocket();
nix::connect(toSocket(conn->fd.get()), path);
nix::connect(toSocket(conn->fd.get()), config->path);
conn->from.fd = conn->fd.get();
conn->to.fd = conn->fd.get();
@ -106,6 +105,11 @@ void UDSRemoteStore::addIndirectRoot(const Path & path)
}
static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regUDSRemoteStore;
ref<Store> UDSRemoteStore::Config::openStore() const {
return make_ref<UDSRemoteStore>(ref{shared_from_this()});
}
static RegisterStoreImplementation<UDSRemoteStore::Config> regUDSRemoteStore;
}

View File

@ -38,7 +38,7 @@
#include "store-config-private.hh"
#if HAVE_STATVFS
#include <sys/statvfs.h>
# include <sys/statvfs.h>
#endif
/* Includes required for chroot support. */
@ -62,9 +62,9 @@
#endif
#ifdef __APPLE__
#include <spawn.h>
#include <sys/sysctl.h>
#include <sandbox.h>
# include <spawn.h>
# include <sys/sysctl.h>
# include <sandbox.h>
/* This definition is undocumented but depended upon by all major browsers. */
extern "C" int sandbox_init_with_parameters(const char *profile, uint64_t flags, const char *const parameters[], char **errorbuf);
@ -512,7 +512,7 @@ Goal::Co LocalDerivationGoal::tryLocalBuild()
}
auto & localStore = getLocalStore();
if (localStore.storeDir != localStore.realStoreDir.get()) {
if (localStore.storeDir != localStore.config->realStoreDir.get()) {
#ifdef __linux__
useChroot = true;
#else
@ -721,7 +721,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
auto & localStore = getLocalStore();
uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
struct statvfs st;
if (statvfs(localStore.realStoreDir.get().c_str(), &st) == 0 &&
if (statvfs(localStore.config->realStoreDir.get().c_str(), &st) == 0 &&
(uint64_t) st.f_bavail * st.f_bsize < required)
diskFull = true;
if (statvfs(tmpDir.c_str(), &st) == 0 &&
@ -885,7 +885,7 @@ void LocalDerivationGoal::startBuilder()
concatStringsSep(", ", drvOptions->getRequiredSystemFeatures(*drv)),
worker.store.printStorePath(drvPath),
settings.thisSystem,
concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
concatStringsSep<StringSet>(", ", worker.store.config.systemFeatures));
}
}
@ -1619,14 +1619,14 @@ void LocalDerivationGoal::startDaemon()
{
experimentalFeatureSettings.require(Xp::RecursiveNix);
Store::Params params;
params["path-info-cache-size"] = "0";
params["store"] = worker.store.storeDir;
if (auto & optRoot = getLocalStore().rootDir.get())
params["root"] = *optRoot;
params["state"] = "/no-such-path";
params["log"] = "/no-such-path";
auto store = makeRestrictedStore(params,
auto store = makeRestrictedStore(
[&]{
auto config = make_ref<LocalStore::Config>(*getLocalStore().config);
config->pathInfoCacheSize.value = 0;
config->stateDir.value = "/no-such-path";
config->logDir.value = "/no-such-path";
return config;
}(),
ref<LocalStore>(std::dynamic_pointer_cast<LocalStore>(worker.store.shared_from_this())),
*this);
@ -1968,7 +1968,7 @@ void LocalDerivationGoal::runChild()
createDirs(chrootRootDir + "/dev/shm");
createDirs(chrootRootDir + "/dev/pts");
ss.push_back("/dev/full");
if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
if (worker.store.config.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
ss.push_back("/dev/kvm");
ss.push_back("/dev/null");
ss.push_back("/dev/random");

View File

@ -0,0 +1,51 @@
#pragma once
///@type
#include <optional>
namespace nix::config {
template<typename T>
struct JustValue
{
T value;
operator const T &() const
{
return value;
}
operator T &()
{
return value;
}
const T & get() const
{
return value;
}
bool operator==(auto && v2) const
{
return value == v2;
}
bool operator!=(auto && v2) const
{
return value != v2;
}
};
template<typename T>
auto && operator<<(auto && str, const JustValue<T> & opt)
{
return str << opt.get();
}
template<typename T>
struct OptValue
{
std::optional<T> optValue;
};
}

View File

@ -180,13 +180,12 @@ public:
const std::string name;
const std::string description;
const std::set<std::string> aliases;
std::optional<ExperimentalFeature> experimentalFeature;
int created = 123;
bool overridden = false;
std::optional<ExperimentalFeature> experimentalFeature;
protected:
AbstractSetting(

View File

@ -16,6 +16,7 @@ headers = files(
'comparator.hh',
'compression.hh',
'compute-levels.hh',
'config-abstract.hh',
'config-global.hh',
'config-impl.hh',
'configuration.hh',

View File

@ -218,16 +218,16 @@ std::pair<std::string_view, std::string_view> getLine(std::string_view s);
/**
* Get a value for the specified key from an associate container.
*/
template <class T>
const typename T::mapped_type * get(const T & map, const typename T::key_type & key)
template <class T, typename K = const T::key_type &>
const typename T::mapped_type * get(const T & map, K key)
{
auto i = map.find(key);
if (i == map.end()) return nullptr;
return &i->second;
}
template <class T>
typename T::mapped_type * get(T & map, const typename T::key_type & key)
template <class T, typename K = const T::key_type &>
typename T::mapped_type * get(T & map, K key)
{
auto i = map.find(key);
if (i == map.end()) return nullptr;
@ -237,9 +237,9 @@ typename T::mapped_type * get(T & map, const typename T::key_type & key)
/**
* Get a value for the specified key from an associate container, or a default value if the key isn't present.
*/
template <class T>
template <class T, typename K = const T::key_type &>
const typename T::mapped_type & getOr(T & map,
const typename T::key_type & key,
K key,
const typename T::mapped_type & defaultValue)
{
auto i = map.find(key);

View File

@ -12,7 +12,7 @@
#include "nix/util/current-process.hh"
#include "nix/store/parsed-derivations.hh"
#include "nix/store/derivation-options.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/store/local-fs-store.hh"
#include "nix/store/globals.hh"
#include "nix/store/realisation.hh"

View File

@ -2,7 +2,7 @@
#include "nix/main/shared.hh"
#include "nix/store/globals.hh"
#include "nix/store/filetransfer.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/cmd/legacy.hh"
#include "nix/expr/eval-settings.hh" // for defexpr
#include "nix/util/users.hh"

View File

@ -1,6 +1,6 @@
#include "nix/util/file-system.hh"
#include "nix/util/signals.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/store/store-cast.hh"
#include "nix/store/gc-store.hh"
#include "nix/store/profiles.hh"

View File

@ -1,6 +1,6 @@
#include "nix/main/shared.hh"
#include "nix/store/realisation.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/cmd/legacy.hh"
#include "man-pages.hh"

View File

@ -9,7 +9,7 @@
#include "nix/store/profiles.hh"
#include "nix/store/path-with-outputs.hh"
#include "nix/main/shared.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/store/local-fs-store.hh"
#include "user-env.hh"
#include "nix/expr/value-to-json.hh"

View File

@ -8,7 +8,7 @@
#include "nix/util/signals.hh"
#include "nix/expr/value-to-xml.hh"
#include "nix/expr/value-to-json.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/store/local-fs-store.hh"
#include "nix/cmd/common-eval-args.hh"
#include "nix/cmd/legacy.hh"

View File

@ -2,6 +2,7 @@
#include "nix/store/derivations.hh"
#include "dotgraph.hh"
#include "nix/store/globals.hh"
#include "nix/store/store-open.hh"
#include "nix/store/store-cast.hh"
#include "nix/store/local-fs-store.hh"
#include "nix/store/log-store.hh"

View File

@ -8,7 +8,7 @@
#include "nix/flake/flake.hh"
#include "nix/expr/get-drvs.hh"
#include "nix/util/signals.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/store/derivations.hh"
#include "nix/store/outputs-spec.hh"
#include "nix/expr/attr-path.hh"

View File

@ -1,7 +1,7 @@
#include "nix/cmd/command.hh"
#include "nix/main/common-args.hh"
#include "nix/main/shared.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/store/log-store.hh"
using namespace nix;

View File

@ -7,7 +7,8 @@
#include "nix/store/globals.hh"
#include "nix/cmd/legacy.hh"
#include "nix/main/shared.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/store/store-registration.hh"
#include "nix/store/filetransfer.hh"
#include "nix/util/finally.hh"
#include "nix/main/loggers.hh"
@ -193,13 +194,12 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
res["args"] = toJSON();
auto stores = nlohmann::json::object();
for (auto & implem : *Implementations::registered) {
auto storeConfig = implem.getConfig();
auto storeName = storeConfig->name();
for (auto & [storeName, implem] : *Implementations::registered) {
auto & j = stores[storeName];
j["doc"] = storeConfig->doc();
j["settings"] = storeConfig->toJSON();
j["experimentalFeature"] = storeConfig->experimentalFeature();
j["doc"] = implem.doc;
j["uri-schemes"] = implem.uriSchemes;
j["settings"] = implem.configDescriptions();
j["experimentalFeature"] = implem.experimentalFeature;
}
res["stores"] = std::move(stores);
res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo();

View File

@ -1,5 +1,5 @@
#include "nix/cmd/command.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/store/make-content-addressed.hh"
#include "nix/main/common-args.hh"

View File

@ -1,7 +1,7 @@
#include "nix/cmd/command.hh"
#include "nix/main/common-args.hh"
#include "nix/main/shared.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/store/filetransfer.hh"
#include "nix/util/finally.hh"
#include "nix/main/loggers.hh"

View File

@ -2,6 +2,7 @@
#include "nix/expr/eval-settings.hh"
#include "nix/util/config-global.hh"
#include "nix/store/globals.hh"
#include "nix/store/store-open.hh"
#include "nix/cmd/command.hh"
#include "nix/cmd/installable-value.hh"
#include "nix/cmd/repl.hh"

Some files were not shown because too many files have changed in this diff Show More