diff --git a/.clang-format b/.clang-format index 73eac7ef6..f5d7fb711 100644 --- a/.clang-format +++ b/.clang-format @@ -15,7 +15,7 @@ SpaceAfterCStyleCast: true SpaceAfterTemplateKeyword: false AccessModifierOffset: -4 AlignAfterOpenBracket: AlwaysBreak -AlignEscapedNewlines: DontAlign +AlignEscapedNewlines: Left ColumnLimit: 120 BreakStringLiterals: false BitFieldColonSpacing: None @@ -30,3 +30,5 @@ BreakBeforeBinaryOperators: NonAssignment AlwaysBreakBeforeMultilineStrings: true IndentPPDirectives: AfterHash PPIndentWidth: 2 +BinPackArguments: false +BinPackParameters: false diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 351a01fcb..f7f05d38e 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -17,7 +17,8 @@ excludes = [ # We don't want to format test data # ''tests/(?!nixos/).*\.nix'' - ''^tests/.*'' + ''^tests/functional/.*$'' + ''^tests/unit/[^/]*/data/.*$'' # Don't format vendored code ''^src/toml11/.*'' @@ -426,6 +427,69 @@ ''^src/nix/upgrade-nix\.cc$'' ''^src/nix/verify\.cc$'' ''^src/nix/why-depends\.cc$'' + + ''^tests/nixos/ca-fd-leak/sender\.c'' + ''^tests/nixos/ca-fd-leak/smuggler\.c'' + ''^tests/unit/libexpr-support/tests/libexpr\.hh'' + ''^tests/unit/libexpr-support/tests/value/context\.cc'' + ''^tests/unit/libexpr-support/tests/value/context\.hh'' + ''^tests/unit/libexpr/derived-path\.cc'' + ''^tests/unit/libexpr/error_traces\.cc'' + ''^tests/unit/libexpr/eval\.cc'' + ''^tests/unit/libexpr/flake/flakeref\.cc'' + ''^tests/unit/libexpr/flake/url-name\.cc'' + ''^tests/unit/libexpr/json\.cc'' + ''^tests/unit/libexpr/main\.cc'' + ''^tests/unit/libexpr/primops\.cc'' + ''^tests/unit/libexpr/search-path\.cc'' + ''^tests/unit/libexpr/trivial\.cc'' + ''^tests/unit/libexpr/value/context\.cc'' + ''^tests/unit/libexpr/value/print\.cc'' + ''^tests/unit/libfetchers/public-key\.cc'' + ''^tests/unit/libstore-support/tests/derived-path\.cc'' + ''^tests/unit/libstore-support/tests/derived-path\.hh'' + ''^tests/unit/libstore-support/tests/libstore\.hh'' + ''^tests/unit/libstore-support/tests/nix_api_store\.hh'' + ''^tests/unit/libstore-support/tests/outputs-spec\.cc'' + ''^tests/unit/libstore-support/tests/outputs-spec\.hh'' + ''^tests/unit/libstore-support/tests/path\.cc'' + ''^tests/unit/libstore-support/tests/path\.hh'' + ''^tests/unit/libstore-support/tests/protocol\.hh'' + ''^tests/unit/libstore/common-protocol\.cc'' + ''^tests/unit/libstore/content-address\.cc'' + ''^tests/unit/libstore/derivation\.cc'' + ''^tests/unit/libstore/derived-path\.cc'' + ''^tests/unit/libstore/downstream-placeholder\.cc'' + ''^tests/unit/libstore/machines\.cc'' + ''^tests/unit/libstore/nar-info-disk-cache\.cc'' + ''^tests/unit/libstore/nar-info\.cc'' + ''^tests/unit/libstore/outputs-spec\.cc'' + ''^tests/unit/libstore/path-info\.cc'' + ''^tests/unit/libstore/path\.cc'' + ''^tests/unit/libstore/serve-protocol\.cc'' + ''^tests/unit/libstore/worker-protocol\.cc'' + ''^tests/unit/libutil-support/tests/characterization\.hh'' + ''^tests/unit/libutil-support/tests/hash\.cc'' + ''^tests/unit/libutil-support/tests/hash\.hh'' + ''^tests/unit/libutil/args\.cc'' + ''^tests/unit/libutil/canon-path\.cc'' + ''^tests/unit/libutil/chunked-vector\.cc'' + ''^tests/unit/libutil/closure\.cc'' + ''^tests/unit/libutil/compression\.cc'' + ''^tests/unit/libutil/config\.cc'' + ''^tests/unit/libutil/file-content-address\.cc'' + ''^tests/unit/libutil/git\.cc'' + ''^tests/unit/libutil/hash\.cc'' + ''^tests/unit/libutil/hilite\.cc'' + ''^tests/unit/libutil/json-utils\.cc'' + ''^tests/unit/libutil/logging\.cc'' + ''^tests/unit/libutil/lru-cache\.cc'' + ''^tests/unit/libutil/pool\.cc'' + ''^tests/unit/libutil/references\.cc'' + ''^tests/unit/libutil/suggestions\.cc'' + ''^tests/unit/libutil/tests\.cc'' + ''^tests/unit/libutil/url\.cc'' + ''^tests/unit/libutil/xml-writer\.cc'' ]; }; diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 18eee830b..582e6d623 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -37,7 +37,7 @@ static std::string currentLoad; static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) { - return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); + return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true); } static bool allSupportedLocally(Store & store, const std::set& requiredFeatures) { @@ -99,7 +99,7 @@ static int main_build_remote(int argc, char * * argv) } std::optional drvPath; - std::string storeUri; + StoreReference storeUri; while (true) { @@ -135,7 +135,7 @@ static int main_build_remote(int argc, char * * argv) Machine * bestMachine = nullptr; uint64_t bestLoad = 0; for (auto & m : machines) { - debug("considering building on remote machine '%s'", m.storeUri); + debug("considering building on remote machine '%s'", m.storeUri.render()); if (m.enabled && m.systemSupported(neededSystem) && @@ -233,7 +233,7 @@ static int main_build_remote(int argc, char * * argv) try { - Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri.render())); sshStore = bestMachine->openStore(); sshStore->connect(); @@ -242,7 +242,7 @@ static int main_build_remote(int argc, char * * argv) } catch (std::exception & e) { auto msg = chomp(drainFD(5, false)); printError("cannot build on '%s': %s%s", - bestMachine->storeUri, e.what(), + bestMachine->storeUri.render(), e.what(), msg.empty() ? "" : ": " + msg); bestMachine->enabled = false; continue; @@ -257,15 +257,15 @@ connected: assert(sshStore); - std::cerr << "# accept\n" << storeUri << "\n"; + std::cerr << "# accept\n" << storeUri.render() << "\n"; auto inputs = readStrings(source); auto wantedOutputs = readStrings(source); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); + AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri.render()) + ".upload-lock", true); { - Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri.render())); auto old = signal(SIGALRM, handleAlarm); alarm(15 * 60); @@ -278,7 +278,7 @@ connected: auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute; { - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri.render())); copyPaths(*store, *sshStore, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute); } @@ -316,7 +316,7 @@ connected: optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv); auto & result = *optResult; if (!result.success()) - throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); + throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri.render(), result.errorMsg); } else { copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute); auto res = sshStore->buildPathsWithResults({ @@ -359,7 +359,7 @@ connected: } if (!missingPaths.empty()) { - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri.render())); if (auto localStore = store.dynamic_pointer_cast()) for (auto & path : missingPaths) localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */ diff --git a/src/libcmd/network-proxy.cc b/src/libcmd/network-proxy.cc index 633b2c005..4b7d2441f 100644 --- a/src/libcmd/network-proxy.cc +++ b/src/libcmd/network-proxy.cc @@ -25,7 +25,10 @@ static StringSet getExcludingNoProxyVariables() static const StringSet excludeVariables{"no_proxy", "NO_PROXY"}; StringSet variables; std::set_difference( - networkProxyVariables.begin(), networkProxyVariables.end(), excludeVariables.begin(), excludeVariables.end(), + networkProxyVariables.begin(), + networkProxyVariables.end(), + excludeVariables.begin(), + excludeVariables.end(), std::inserter(variables, variables.begin())); return variables; } diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 2d461c63a..644762242 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -6,7 +6,8 @@ namespace nix { -Machine::Machine(decltype(storeUri) storeUri, +Machine::Machine( + const std::string & storeUri, decltype(systemTypes) systemTypes, decltype(sshKey) sshKey, decltype(maxJobs) maxJobs, @@ -14,7 +15,7 @@ Machine::Machine(decltype(storeUri) storeUri, decltype(supportedFeatures) supportedFeatures, decltype(mandatoryFeatures) mandatoryFeatures, decltype(sshPublicHostKey) sshPublicHostKey) : - storeUri( + storeUri(StoreReference::parse( // Backwards compatibility: if the URI is schemeless, is not a path, // and is not one of the special store connection words, prepend // ssh://. @@ -28,7 +29,7 @@ Machine::Machine(decltype(storeUri) storeUri, || hasPrefix(storeUri, "local?") || hasPrefix(storeUri, "?") ? storeUri - : "ssh://" + storeUri), + : "ssh://" + storeUri)), systemTypes(systemTypes), sshKey(sshKey), maxJobs(maxJobs), @@ -63,23 +64,26 @@ bool Machine::mandatoryMet(const std::set & features) const }); } -ref Machine::openStore() const +StoreReference Machine::completeStoreReference() const { - Store::Params storeParams; - if (hasPrefix(storeUri, "ssh://")) { - storeParams["max-connections"] = "1"; - storeParams["log-fd"] = "4"; + auto storeUri = this->storeUri; + + auto * generic = std::get_if(&storeUri.variant); + + if (generic && generic->scheme == "ssh") { + storeUri.params["max-connections"] = "1"; + storeUri.params["log-fd"] = "4"; } - if (hasPrefix(storeUri, "ssh://") || hasPrefix(storeUri, "ssh-ng://")) { + if (generic && (generic->scheme == "ssh" || generic->scheme == "ssh-ng")) { if (sshKey != "") - storeParams["ssh-key"] = sshKey; + storeUri.params["ssh-key"] = sshKey; if (sshPublicHostKey != "") - storeParams["base64-ssh-public-host-key"] = sshPublicHostKey; + storeUri.params["base64-ssh-public-host-key"] = sshPublicHostKey; } { - auto & fs = storeParams["system-features"]; + auto & fs = storeUri.params["system-features"]; auto append = [&](auto feats) { for (auto & f : feats) { if (fs.size() > 0) fs += ' '; @@ -90,7 +94,12 @@ ref Machine::openStore() const append(mandatoryFeatures); } - return nix::openStore(storeUri, storeParams); + return storeUri; +} + +ref Machine::openStore() const +{ + return nix::openStore(completeStoreReference()); } static std::vector expandBuilderLines(const std::string & builders) diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh index 8516409d4..97980df8d 100644 --- a/src/libstore/machines.hh +++ b/src/libstore/machines.hh @@ -2,6 +2,7 @@ ///@file #include "types.hh" +#include "store-reference.hh" namespace nix { @@ -9,7 +10,7 @@ class Store; struct Machine { - const std::string storeUri; + const StoreReference storeUri; const std::set systemTypes; const std::string sshKey; const unsigned int maxJobs; @@ -36,7 +37,8 @@ struct Machine { */ bool mandatoryMet(const std::set & features) const; - Machine(decltype(storeUri) storeUri, + Machine( + const std::string & storeUri, decltype(systemTypes) systemTypes, decltype(sshKey) sshKey, decltype(maxJobs) maxJobs, @@ -45,6 +47,21 @@ struct Machine { decltype(mandatoryFeatures) mandatoryFeatures, decltype(sshPublicHostKey) sshPublicHostKey); + /** + * Elaborate `storeUri` into a complete store reference, + * incorporating information from the other fields of the `Machine` + * as applicable. + */ + StoreReference completeStoreReference() const; + + /** + * Open a `Store` for this machine. + * + * Just a simple function composition: + * ```c++ + * nix::openStore(completeStoreReference()) + * ``` + */ ref openStore() const; }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 6eba3a77d..7c2b3815f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -8,7 +8,6 @@ #include "util.hh" #include "nar-info-disk-cache.hh" #include "thread-pool.hh" -#include "url.hh" #include "references.hh" #include "archive.hh" #include "callback.hh" @@ -1267,109 +1266,63 @@ Derivation Store::readInvalidDerivation(const StorePath & drvPath) namespace nix { -/* Split URI into protocol+hierarchy part and its parameter set. */ -std::pair splitUriAndParams(const std::string & uri_) -{ - auto uri(uri_); - Store::Params params; - auto q = uri.find('?'); - if (q != std::string::npos) { - params = decodeQuery(uri.substr(q + 1)); - uri = uri_.substr(0, q); - } - return {uri, params}; -} - -static bool isNonUriPath(const std::string & spec) -{ - return - // is not a URL - spec.find("://") == std::string::npos - // Has at least one path separator, and so isn't a single word that - // might be special like "auto" - && spec.find("/") != std::string::npos; -} - -std::shared_ptr openFromNonUri(const std::string & uri, const Store::Params & params) -{ - // TODO reenable on Windows once we have `LocalStore` and - // `UDSRemoteStore`. - if (uri == "" || uri == "auto") { - auto stateDir = getOr(params, "state", settings.nixStateDir); - if (access(stateDir.c_str(), R_OK | W_OK) == 0) - return std::make_shared(params); - else if (pathExists(settings.nixDaemonSocketFile)) - return std::make_shared(params); - #if __linux__ - else if (!pathExists(stateDir) - && params.empty() - && !isRootUser() - && !getEnv("NIX_STORE_DIR").has_value() - && !getEnv("NIX_STATE_DIR").has_value()) - { - /* If /nix doesn't exist, there is no daemon socket, and - we're not root, then automatically set up a chroot - store in ~/.local/share/nix/root. */ - auto chrootStore = getDataDir() + "/nix/root"; - if (!pathExists(chrootStore)) { - try { - createDirs(chrootStore); - } catch (Error & e) { - return std::make_shared(params); - } - warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); - } else - debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); - return std::make_shared("local", chrootStore, params); - } - #endif - else - return std::make_shared(params); - } else if (uri == "daemon") { - return std::make_shared(params); - } else if (uri == "local") { - return std::make_shared(params); - } else if (isNonUriPath(uri)) { - return std::make_shared("local", absPath(uri), params); - } else { - return nullptr; - } -} - -ref openStore(const std::string & uri_, +ref openStore(const std::string & uri, const Store::Params & extraParams) { - auto params = extraParams; - try { - auto parsedUri = parseURL(uri_); - params.insert(parsedUri.query.begin(), parsedUri.query.end()); + return openStore(StoreReference::parse(uri, extraParams)); +} - auto baseURI = parsedUri.authority.value_or("") + parsedUri.path; +ref openStore(StoreReference && storeURI) +{ + auto & params = storeURI.params; - for (auto implem : *Implementations::registered) { - if (implem.uriSchemes.count(parsedUri.scheme)) { - auto store = implem.create(parsedUri.scheme, baseURI, params); - if (store) { - experimentalFeatureSettings.require(store->experimentalFeature()); - store->init(); - store->warnUnknownSettings(); - return ref(store); - } + auto store = std::visit(overloaded { + [&](const StoreReference::Auto &) -> std::shared_ptr { + auto stateDir = getOr(params, "state", settings.nixStateDir); + if (access(stateDir.c_str(), R_OK | W_OK) == 0) + return std::make_shared(params); + else if (pathExists(settings.nixDaemonSocketFile)) + return std::make_shared(params); + #if __linux__ + else if (!pathExists(stateDir) + && params.empty() + && !isRootUser() + && !getEnv("NIX_STORE_DIR").has_value() + && !getEnv("NIX_STATE_DIR").has_value()) + { + /* If /nix doesn't exist, there is no daemon socket, and + we're not root, then automatically set up a chroot + store in ~/.local/share/nix/root. */ + auto chrootStore = getDataDir() + "/nix/root"; + if (!pathExists(chrootStore)) { + try { + createDirs(chrootStore); + } catch (Error & e) { + return std::make_shared(params); + } + warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); + } else + debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); + return std::make_shared("local", chrootStore, params); } - } - } - catch (BadURL &) { - auto [uri, uriParams] = splitUriAndParams(uri_); - params.insert(uriParams.begin(), uriParams.end()); + #endif + else + return std::make_shared(params); + }, + [&](const StoreReference::Specified & g) { + for (auto implem : *Implementations::registered) + if (implem.uriSchemes.count(g.scheme)) + return implem.create(g.scheme, g.authority, params); - if (auto store = openFromNonUri(uri, params)) { - experimentalFeatureSettings.require(store->experimentalFeature()); - store->warnUnknownSettings(); - return ref(store); - } - } + throw Error("don't know how to open Nix store with scheme '%s'", g.scheme); + }, + }, storeURI.variant); - throw Error("don't know how to open Nix store '%s'", uri_); + experimentalFeatureSettings.require(store->experimentalFeature()); + store->warnUnknownSettings(); + store->init(); + + return ref { store }; } std::list> getDefaultSubstituters() diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index a508e5a00..430d9a5ab 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -13,6 +13,7 @@ #include "path-info.hh" #include "repair-flag.hh" #include "store-dir-config.hh" +#include "store-reference.hh" #include "source-path.hh" #include @@ -65,7 +66,7 @@ MakeError(Unsupported, Error); MakeError(SubstituteGone, Error); MakeError(SubstituterDisabled, Error); -MakeError(InvalidStoreURI, Error); +MakeError(InvalidStoreReference, Error); struct Realisation; struct RealisedPath; @@ -102,7 +103,7 @@ typedef std::map> StorePathCAMap; struct StoreConfig : public StoreDirConfig { - typedef std::map Params; + using Params = StoreReference::Params; using StoreDirConfig::StoreDirConfig; @@ -859,34 +860,13 @@ OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * ev /** * @return a Store object to access the Nix store denoted by * ‘uri’ (slight misnomer...). - * - * @param uri Supported values are: - * - * - ‘local’: The Nix store in /nix/store and database in - * /nix/var/nix/db, accessed directly. - * - * - ‘daemon’: The Nix store accessed via a Unix domain socket - * connection to nix-daemon. - * - * - ‘unix://’: The Nix store accessed via a Unix domain socket - * connection to nix-daemon, with the socket located at . - * - * - ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on - * whether the user has write access to the local Nix - * store/database. - * - * - ‘file://’: A binary cache stored in . - * - * - ‘https://’: A binary cache accessed via HTTP. - * - * - ‘s3://’: A writable binary cache stored on Amazon's Simple - * Storage Service. - * - * - ‘ssh://[user@]’: A remote Nix store accessed by running - * ‘nix-store --serve’ via SSH. - * - * You can pass parameters to the store type by appending - * ‘?key=value&key=value&...’ to the URI. + */ +ref openStore(StoreReference && storeURI); + + +/** + * Opens the store at `uri`, where `uri` is in the format expected by `StoreReference::parse` + */ ref openStore(const std::string & uri = settings.storeUri.get(), const Store::Params & extraParams = Store::Params()); @@ -957,11 +937,6 @@ std::optional decodeValidPathInfo( std::istream & str, std::optional hashGiven = std::nullopt); -/** - * Split URI into protocol+hierarchy part and its parameter set. - */ -std::pair splitUriAndParams(const std::string & uri); - const ContentAddress * getDerivationCA(const BasicDerivation & drv); std::map drvOutputReferences( diff --git a/src/libstore/store-reference.cc b/src/libstore/store-reference.cc new file mode 100644 index 000000000..b4968dfad --- /dev/null +++ b/src/libstore/store-reference.cc @@ -0,0 +1,116 @@ +#include + +#include "error.hh" +#include "url.hh" +#include "store-reference.hh" +#include "file-system.hh" +#include "util.hh" + +namespace nix { + +static bool isNonUriPath(const std::string & spec) +{ + return + // is not a URL + spec.find("://") == std::string::npos + // Has at least one path separator, and so isn't a single word that + // might be special like "auto" + && spec.find("/") != std::string::npos; +} + +std::string StoreReference::render() const +{ + std::string res; + + std::visit( + overloaded{ + [&](const StoreReference::Auto &) { res = "auto"; }, + [&](const StoreReference::Specified & g) { + res = g.scheme; + res += "://"; + res += g.authority; + }, + }, + variant); + + if (!params.empty()) { + res += "?"; + res += encodeQuery(params); + } + + return res; +} + +StoreReference StoreReference::parse(const std::string & uri, const StoreReference::Params & extraParams) +{ + auto params = extraParams; + try { + auto parsedUri = parseURL(uri); + params.insert(parsedUri.query.begin(), parsedUri.query.end()); + + auto baseURI = parsedUri.authority.value_or("") + parsedUri.path; + + return { + .variant = + Specified{ + .scheme = std::move(parsedUri.scheme), + .authority = std::move(baseURI), + }, + .params = std::move(params), + }; + } catch (BadURL &) { + auto [baseURI, uriParams] = splitUriAndParams(uri); + params.insert(uriParams.begin(), uriParams.end()); + + if (baseURI == "" || baseURI == "auto") { + return { + .variant = Auto{}, + .params = std::move(params), + }; + } else if (baseURI == "daemon") { + return { + .variant = + Specified{ + .scheme = "unix", + .authority = "", + }, + .params = std::move(params), + }; + } else if (baseURI == "local") { + return { + .variant = + Specified{ + .scheme = "local", + .authority = "", + }, + .params = std::move(params), + }; + } else if (isNonUriPath(baseURI)) { + return { + .variant = + Specified{ + .scheme = "local", + .authority = absPath(baseURI), + }, + .params = std::move(params), + }; + } + } + + throw UsageError("Cannot parse Nix store '%s'", uri); +} + +/* Split URI into protocol+hierarchy part and its parameter set. */ +std::pair splitUriAndParams(const std::string & uri_) +{ + auto uri(uri_); + StoreReference::Params params; + auto q = uri.find('?'); + if (q != std::string::npos) { + params = decodeQuery(uri.substr(q + 1)); + uri = uri_.substr(0, q); + } + return {uri, params}; +} + +} diff --git a/src/libstore/store-reference.hh b/src/libstore/store-reference.hh new file mode 100644 index 000000000..e99335c0d --- /dev/null +++ b/src/libstore/store-reference.hh @@ -0,0 +1,92 @@ +#pragma once +///@file + +#include + +#include "types.hh" + +namespace nix { + +/** + * A parsed Store URI (URI is a slight misnomer...), parsed but not yet + * resolved to a specific instance and query parms validated. + * + * Supported values are: + * + * - ‘local’: The Nix store in /nix/store and database in + * /nix/var/nix/db, accessed directly. + * + * - ‘daemon’: The Nix store accessed via a Unix domain socket + * connection to nix-daemon. + * + * - ‘unix://’: The Nix store accessed via a Unix domain socket + * connection to nix-daemon, with the socket located at . + * + * - ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on + * whether the user has write access to the local Nix + * store/database. + * + * - ‘file://’: A binary cache stored in . + * + * - ‘https://’: A binary cache accessed via HTTP. + * + * - ‘s3://’: A writable binary cache stored on Amazon's Simple + * Storage Service. + * + * - ‘ssh://[user@]’: A remote Nix store accessed by running + * ‘nix-store --serve’ via SSH. + * + * You can pass parameters to the store type by appending + * ‘?key=value&key=value&...’ to the URI. + */ +struct StoreReference +{ + using Params = std::map; + + /** + * Special store reference `""` or `"auto"` + */ + struct Auto + { + inline bool operator==(const Auto & rhs) const = default; + inline auto operator<=>(const Auto & rhs) const = default; + }; + + /** + * General case, a regular `scheme://authority` URL. + */ + struct Specified + { + std::string scheme; + std::string authority = ""; + + bool operator==(const Specified & rhs) const = default; + auto operator<=>(const Specified & rhs) const = default; + }; + + typedef std::variant Variant; + + Variant variant; + + Params params; + + bool operator==(const StoreReference & rhs) const = default; + auto operator<=>(const StoreReference & rhs) const = default; + + /** + * Render the whole store reference as a URI, including parameters. + */ + std::string render() const; + + /** + * Parse a URI into a store reference. + */ + static StoreReference parse(const std::string & uri, const Params & extraParams = Params{}); +}; + +/** + * Split URI into protocol+hierarchy part and its parameter set. + */ +std::pair splitUriAndParams(const std::string & uri); + +} diff --git a/src/libstore/windows/pathlocks.cc b/src/libstore/windows/pathlocks.cc index 738057f68..1199878e9 100644 --- a/src/libstore/windows/pathlocks.cc +++ b/src/libstore/windows/pathlocks.cc @@ -35,8 +35,13 @@ void PathLocks::unlock() AutoCloseFD openLockFile(const Path & path, bool create) { AutoCloseFD desc = CreateFileA( - path.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, - create ? OPEN_ALWAYS : OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_POSIX_SEMANTICS, NULL); + path.c_str(), + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, + NULL, + create ? OPEN_ALWAYS : OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL | FILE_FLAG_POSIX_SEMANTICS, + NULL); if (desc.get() == INVALID_HANDLE_VALUE) warn("%s: %s", path, std::to_string(GetLastError())); diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index d17401f27..d27028565 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -263,8 +263,13 @@ struct BrotliCompressionSink : ChunkedCompressionSink checkInterrupt(); if (!BrotliEncoderCompressStream( - state, data.data() ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH, &avail_in, &next_in, - &avail_out, &next_out, nullptr)) + state, + data.data() ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH, + &avail_in, + &next_in, + &avail_out, + &next_out, + nullptr)) throw CompressionError("error while compressing brotli compression"); if (avail_out < sizeof(outbuf) || avail_in == 0) { @@ -280,8 +285,8 @@ struct BrotliCompressionSink : ChunkedCompressionSink ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel, int level) { - std::vector la_supports = {"bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", - "lzip", "lzma", "lzop", "xz", "zstd"}; + std::vector la_supports = { + "bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd"}; if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) { return make_ref(nextSink, method, parallel, level); } diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 24806bbff..6cd06e53d 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -33,6 +33,8 @@ std::string percentEncode(std::string_view s, std::string_view keep=""); std::map decodeQuery(const std::string & query); +std::string encodeQuery(const std::map & query); + ParsedURL parseURL(const std::string & url); /** diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index 8002dd75e..b15355efe 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -5,8 +5,13 @@ namespace nix { Descriptor openDirectory(const std::filesystem::path & path) { return CreateFileW( - path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS, NULL); + path.c_str(), + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + NULL); } } diff --git a/tests/unit/libexpr/nix_api_value.cc b/tests/unit/libexpr/nix_api_value.cc index 6e1131e10..c71593c85 100644 --- a/tests/unit/libexpr/nix_api_value.cc +++ b/tests/unit/libexpr/nix_api_value.cc @@ -256,10 +256,13 @@ TEST_F(nix_api_expr_test, nix_value_init) Value * f = nix_alloc_value(ctx, state); nix_expr_eval_from_string( - ctx, state, R"( + ctx, + state, + R"( a: a * a )", - "", f); + "", + f); // Test @@ -325,20 +328,26 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg) Value * f = nix_alloc_value(ctx, state); nix_expr_eval_from_string( - ctx, state, R"( + ctx, + state, + R"( a: { foo = a; } )", - "", f); + "", + f); assert_ctx_ok(); Value * e = nix_alloc_value(ctx, state); { Value * g = nix_alloc_value(ctx, state); nix_expr_eval_from_string( - ctx, state, R"( + ctx, + state, + R"( _ignore: throw "error message for test case nix_value_init_apply_lazy_arg" )", - "", g); + "", + g); assert_ctx_ok(); nix_init_apply(ctx, e, g, g); diff --git a/tests/unit/libstore/data/store-reference/auto.txt b/tests/unit/libstore/data/store-reference/auto.txt new file mode 100644 index 000000000..4d18c3e59 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/auto.txt @@ -0,0 +1 @@ +auto \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/auto_param.txt b/tests/unit/libstore/data/store-reference/auto_param.txt new file mode 100644 index 000000000..54adabb25 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/auto_param.txt @@ -0,0 +1 @@ +auto?root=/foo/bar/baz \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/local_1.txt b/tests/unit/libstore/data/store-reference/local_1.txt new file mode 100644 index 000000000..74b1b9677 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/local_1.txt @@ -0,0 +1 @@ +local://?root=/foo/bar/baz \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/local_2.txt b/tests/unit/libstore/data/store-reference/local_2.txt new file mode 100644 index 000000000..8b5593fb1 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/local_2.txt @@ -0,0 +1 @@ +local:///foo/bar/baz?trusted=true \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/local_shorthand_1.txt b/tests/unit/libstore/data/store-reference/local_shorthand_1.txt new file mode 100644 index 000000000..896189be9 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/local_shorthand_1.txt @@ -0,0 +1 @@ +local?root=/foo/bar/baz \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/local_shorthand_2.txt b/tests/unit/libstore/data/store-reference/local_shorthand_2.txt new file mode 100644 index 000000000..7a9dad3b3 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/local_shorthand_2.txt @@ -0,0 +1 @@ +/foo/bar/baz?trusted=true \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/ssh.txt b/tests/unit/libstore/data/store-reference/ssh.txt new file mode 100644 index 000000000..8c61010ec --- /dev/null +++ b/tests/unit/libstore/data/store-reference/ssh.txt @@ -0,0 +1 @@ +ssh://localhost \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/unix.txt b/tests/unit/libstore/data/store-reference/unix.txt new file mode 100644 index 000000000..195489048 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/unix.txt @@ -0,0 +1 @@ +unix://?max-connections=7&trusted=true \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/unix_shorthand.txt b/tests/unit/libstore/data/store-reference/unix_shorthand.txt new file mode 100644 index 000000000..0300337e9 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/unix_shorthand.txt @@ -0,0 +1 @@ +daemon?max-connections=7&trusted=true \ No newline at end of file diff --git a/tests/unit/libstore/machines.cc b/tests/unit/libstore/machines.cc index 9fd7fda54..4107ba655 100644 --- a/tests/unit/libstore/machines.cc +++ b/tests/unit/libstore/machines.cc @@ -3,24 +3,16 @@ #include "file-system.hh" #include "util.hh" +#include #include using testing::Contains; using testing::ElementsAre; -using testing::EndsWith; using testing::Eq; using testing::Field; using testing::SizeIs; -using nix::absPath; -using nix::FormatError; -using nix::UsageError; -using nix::getMachines; -using nix::Machine; -using nix::Machines; -using nix::pathExists; -using nix::Settings; -using nix::settings; +using namespace nix; class Environment : public ::testing::Environment { public: @@ -40,7 +32,7 @@ TEST(machines, getMachinesUriOnly) { settings.builders = "nix@scratchy.labs.cs.uu.nl"; Machines actual = getMachines(); ASSERT_THAT(actual, SizeIs(1)); - EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl")))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); @@ -54,7 +46,7 @@ TEST(machines, getMachinesDefaults) { settings.builders = "nix@scratchy.labs.cs.uu.nl - - - - - - -"; Machines actual = getMachines(); ASSERT_THAT(actual, SizeIs(1)); - EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl")))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); @@ -64,20 +56,31 @@ TEST(machines, getMachinesDefaults) { EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, SizeIs(0))); } +MATCHER_P(AuthorityMatches, authority, "") { + *result_listener + << "where the authority of " + << arg.render() + << " is " + << authority; + auto * generic = std::get_if(&arg.variant); + if (!generic) return false; + return generic->authority == authority; +} + TEST(machines, getMachinesWithNewLineSeparator) { settings.builders = "nix@scratchy.labs.cs.uu.nl\nnix@itchy.labs.cs.uu.nl"; Machines actual = getMachines(); ASSERT_THAT(actual, SizeIs(2)); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl")))); } TEST(machines, getMachinesWithSemicolonSeparator) { settings.builders = "nix@scratchy.labs.cs.uu.nl ; nix@itchy.labs.cs.uu.nl"; Machines actual = getMachines(); EXPECT_THAT(actual, SizeIs(2)); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl")))); } TEST(machines, getMachinesWithCorrectCompleteSingleBuilder) { @@ -86,7 +89,7 @@ TEST(machines, getMachinesWithCorrectCompleteSingleBuilder) { "benchmark SSH+HOST+PUBLIC+KEY+BASE64+ENCODED=="; Machines actual = getMachines(); ASSERT_THAT(actual, SizeIs(1)); - EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux"))); EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto"))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8))); @@ -104,7 +107,7 @@ TEST(machines, "KEY+BASE64+ENCODED=="; Machines actual = getMachines(); ASSERT_THAT(actual, SizeIs(1)); - EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux"))); EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto"))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8))); @@ -120,7 +123,7 @@ TEST(machines, getMachinesWithMultiOptions) { "MandatoryFeature1,MandatoryFeature2"; Machines actual = getMachines(); ASSERT_THAT(actual, SizeIs(1)); - EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("Arch1", "Arch2"))); EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("SupportedFeature1", "SupportedFeature2"))); EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("MandatoryFeature1", "MandatoryFeature2"))); @@ -146,9 +149,9 @@ TEST(machines, getMachinesWithCorrectFileReference) { settings.builders = std::string("@") + path; Machines actual = getMachines(); ASSERT_THAT(actual, SizeIs(3)); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@poochie.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@poochie.labs.cs.uu.nl")))); } TEST(machines, getMachinesWithCorrectFileReferenceToEmptyFile) { diff --git a/tests/unit/libstore/store-reference.cc b/tests/unit/libstore/store-reference.cc new file mode 100644 index 000000000..16e033ec4 --- /dev/null +++ b/tests/unit/libstore/store-reference.cc @@ -0,0 +1,123 @@ +#include +#include + +#include "file-system.hh" +#include "store-reference.hh" + +#include "tests/characterization.hh" +#include "tests/libstore.hh" + +namespace nix { + +using nlohmann::json; + +class StoreReferenceTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/store-reference"; + + Path goldenMaster(PathView testStem) const override + { + return unitTestData + "/" + testStem + ".txt"; + } +}; + +#define URI_TEST_READ(STEM, OBJ) \ + TEST_F(StoreReferenceTest, PathInfo_##STEM##_from_uri) \ + { \ + readTest(#STEM, ([&](const auto & encoded) { \ + StoreReference expected = OBJ; \ + auto got = StoreReference::parse(encoded); \ + ASSERT_EQ(got, expected); \ + })); \ + } + +#define URI_TEST_WRITE(STEM, OBJ) \ + TEST_F(StoreReferenceTest, PathInfo_##STEM##_to_uri) \ + { \ + writeTest( \ + #STEM, \ + [&]() -> StoreReference { return OBJ; }, \ + [](const auto & file) { return StoreReference::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.render()); }); \ + } + +#define URI_TEST(STEM, OBJ) \ + URI_TEST_READ(STEM, OBJ) \ + URI_TEST_WRITE(STEM, OBJ) + +URI_TEST( + auto, + (StoreReference{ + .variant = StoreReference::Auto{}, + .params = {}, + })) + +URI_TEST( + auto_param, + (StoreReference{ + .variant = StoreReference::Auto{}, + .params = + { + {"root", "/foo/bar/baz"}, + }, + })) + +static StoreReference localExample_1{ + .variant = + StoreReference::Specified{ + .scheme = "local", + }, + .params = + { + {"root", "/foo/bar/baz"}, + }, +}; + +static StoreReference localExample_2{ + .variant = + StoreReference::Specified{ + .scheme = "local", + .authority = "/foo/bar/baz", + }, + .params = + { + {"trusted", "true"}, + }, +}; + +URI_TEST(local_1, localExample_1) + +URI_TEST(local_2, localExample_2) + +URI_TEST_READ(local_shorthand_1, localExample_1) + +URI_TEST_READ(local_shorthand_2, localExample_2) + +static StoreReference unixExample{ + .variant = + StoreReference::Specified{ + .scheme = "unix", + }, + .params = + { + {"max-connections", "7"}, + {"trusted", "true"}, + }, +}; + +URI_TEST(unix, unixExample) + +URI_TEST_READ(unix_shorthand, unixExample) + +URI_TEST( + ssh, + (StoreReference{ + .variant = + StoreReference::Specified{ + .scheme = "ssh", + .authority = "localhost", + }, + .params = {}, + })) + +}