Merge pull request #9839 from obsidiansystems/more-machine-cleanup

Create `StoreReference` and use it in `Machine`

.clang-format

@@ -15,7 +15,7 @@ SpaceAfterCStyleCast: true
 SpaceAfterTemplateKeyword: false
 AccessModifierOffset: -4
 AlignAfterOpenBracket: AlwaysBreak
-AlignEscapedNewlines: DontAlign
+AlignEscapedNewlines: Left
 ColumnLimit: 120
 BreakStringLiterals: false
 BitFieldColonSpacing: None
@@ -30,3 +30,5 @@ BreakBeforeBinaryOperators: NonAssignment
 AlwaysBreakBeforeMultilineStrings: true
 IndentPPDirectives: AfterHash
 PPIndentWidth: 2
+BinPackArguments: false
+BinPackParameters: false

maintainers/flake-module.nix

@@ -17,7 +17,8 @@
       excludes = [
         # We don't want to format test data
         # ''tests/(?!nixos/).*\.nix''
-        ''^tests/.*''
+        ''^tests/functional/.*$''
+        ''^tests/unit/[^/]*/data/.*$''
 
         # Don't format vendored code
         ''^src/toml11/.*''
@@ -426,6 +427,69 @@
         ''^src/nix/upgrade-nix\.cc$''
         ''^src/nix/verify\.cc$''
         ''^src/nix/why-depends\.cc$''
+
+        ''^tests/nixos/ca-fd-leak/sender\.c''
+        ''^tests/nixos/ca-fd-leak/smuggler\.c''
+        ''^tests/unit/libexpr-support/tests/libexpr\.hh''
+        ''^tests/unit/libexpr-support/tests/value/context\.cc''
+        ''^tests/unit/libexpr-support/tests/value/context\.hh''
+        ''^tests/unit/libexpr/derived-path\.cc''
+        ''^tests/unit/libexpr/error_traces\.cc''
+        ''^tests/unit/libexpr/eval\.cc''
+        ''^tests/unit/libexpr/flake/flakeref\.cc''
+        ''^tests/unit/libexpr/flake/url-name\.cc''
+        ''^tests/unit/libexpr/json\.cc''
+        ''^tests/unit/libexpr/main\.cc''
+        ''^tests/unit/libexpr/primops\.cc''
+        ''^tests/unit/libexpr/search-path\.cc''
+        ''^tests/unit/libexpr/trivial\.cc''
+        ''^tests/unit/libexpr/value/context\.cc''
+        ''^tests/unit/libexpr/value/print\.cc''
+        ''^tests/unit/libfetchers/public-key\.cc''
+        ''^tests/unit/libstore-support/tests/derived-path\.cc''
+        ''^tests/unit/libstore-support/tests/derived-path\.hh''
+        ''^tests/unit/libstore-support/tests/libstore\.hh''
+        ''^tests/unit/libstore-support/tests/nix_api_store\.hh''
+        ''^tests/unit/libstore-support/tests/outputs-spec\.cc''
+        ''^tests/unit/libstore-support/tests/outputs-spec\.hh''
+        ''^tests/unit/libstore-support/tests/path\.cc''
+        ''^tests/unit/libstore-support/tests/path\.hh''
+        ''^tests/unit/libstore-support/tests/protocol\.hh''
+        ''^tests/unit/libstore/common-protocol\.cc''
+        ''^tests/unit/libstore/content-address\.cc''
+        ''^tests/unit/libstore/derivation\.cc''
+        ''^tests/unit/libstore/derived-path\.cc''
+        ''^tests/unit/libstore/downstream-placeholder\.cc''
+        ''^tests/unit/libstore/machines\.cc''
+        ''^tests/unit/libstore/nar-info-disk-cache\.cc''
+        ''^tests/unit/libstore/nar-info\.cc''
+        ''^tests/unit/libstore/outputs-spec\.cc''
+        ''^tests/unit/libstore/path-info\.cc''
+        ''^tests/unit/libstore/path\.cc''
+        ''^tests/unit/libstore/serve-protocol\.cc''
+        ''^tests/unit/libstore/worker-protocol\.cc''
+        ''^tests/unit/libutil-support/tests/characterization\.hh''
+        ''^tests/unit/libutil-support/tests/hash\.cc''
+        ''^tests/unit/libutil-support/tests/hash\.hh''
+        ''^tests/unit/libutil/args\.cc''
+        ''^tests/unit/libutil/canon-path\.cc''
+        ''^tests/unit/libutil/chunked-vector\.cc''
+        ''^tests/unit/libutil/closure\.cc''
+        ''^tests/unit/libutil/compression\.cc''
+        ''^tests/unit/libutil/config\.cc''
+        ''^tests/unit/libutil/file-content-address\.cc''
+        ''^tests/unit/libutil/git\.cc''
+        ''^tests/unit/libutil/hash\.cc''
+        ''^tests/unit/libutil/hilite\.cc''
+        ''^tests/unit/libutil/json-utils\.cc''
+        ''^tests/unit/libutil/logging\.cc''
+        ''^tests/unit/libutil/lru-cache\.cc''
+        ''^tests/unit/libutil/pool\.cc''
+        ''^tests/unit/libutil/references\.cc''
+        ''^tests/unit/libutil/suggestions\.cc''
+        ''^tests/unit/libutil/tests\.cc''
+        ''^tests/unit/libutil/url\.cc''
+        ''^tests/unit/libutil/xml-writer\.cc''
       ];
     };
 

src/build-remote/build-remote.cc

@@ -37,7 +37,7 @@ static std::string currentLoad;
 
 static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
 {
-    return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
+    return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true);
 }
 
 static bool allSupportedLocally(Store & store, const std::set<std::string>& requiredFeatures) {
@@ -99,7 +99,7 @@ static int main_build_remote(int argc, char * * argv)
         }
 
         std::optional<StorePath> drvPath;
-        std::string storeUri;
+        StoreReference storeUri;
 
         while (true) {
 
@@ -135,7 +135,7 @@ static int main_build_remote(int argc, char * * argv)
                 Machine * bestMachine = nullptr;
                 uint64_t bestLoad = 0;
                 for (auto & m : machines) {
-                    debug("considering building on remote machine '%s'", m.storeUri);
+                    debug("considering building on remote machine '%s'", m.storeUri.render());
 
                     if (m.enabled &&
                         m.systemSupported(neededSystem) &&
@@ -233,7 +233,7 @@ static int main_build_remote(int argc, char * * argv)
 
                 try {
 
-                    Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri));
+                    Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri.render()));
 
                     sshStore = bestMachine->openStore();
                     sshStore->connect();
@@ -242,7 +242,7 @@ static int main_build_remote(int argc, char * * argv)
                 } catch (std::exception & e) {
                     auto msg = chomp(drainFD(5, false));
                     printError("cannot build on '%s': %s%s",
-                        bestMachine->storeUri, e.what(),
+                        bestMachine->storeUri.render(), e.what(),
                         msg.empty() ? "" : ": " + msg);
                     bestMachine->enabled = false;
                     continue;
@@ -257,15 +257,15 @@ connected:
 
         assert(sshStore);
 
-        std::cerr << "# accept\n" << storeUri << "\n";
+        std::cerr << "# accept\n" << storeUri.render() << "\n";
 
         auto inputs = readStrings<PathSet>(source);
         auto wantedOutputs = readStrings<StringSet>(source);
 
-        AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
+        AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri.render()) + ".upload-lock", true);
 
         {
-            Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri));
+            Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri.render()));
 
             auto old = signal(SIGALRM, handleAlarm);
             alarm(15 * 60);
@@ -278,7 +278,7 @@ connected:
         auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute;
 
         {
-            Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri));
+            Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri.render()));
             copyPaths(*store, *sshStore, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute);
         }
 
@@ -316,7 +316,7 @@ connected:
             optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv);
             auto & result = *optResult;
             if (!result.success())
-                throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
+                throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri.render(), result.errorMsg);
         } else {
             copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute);
             auto res = sshStore->buildPathsWithResults({
@@ -359,7 +359,7 @@ connected:
         }
 
         if (!missingPaths.empty()) {
-            Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
+            Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri.render()));
             if (auto localStore = store.dynamic_pointer_cast<LocalStore>())
                 for (auto & path : missingPaths)
                     localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */

src/libcmd/network-proxy.cc

@@ -25,7 +25,10 @@ static StringSet getExcludingNoProxyVariables()
     static const StringSet excludeVariables{"no_proxy", "NO_PROXY"};
     StringSet variables;
     std::set_difference(
-        networkProxyVariables.begin(), networkProxyVariables.end(), excludeVariables.begin(), excludeVariables.end(),
+        networkProxyVariables.begin(),
+        networkProxyVariables.end(),
+        excludeVariables.begin(),
+        excludeVariables.end(),
         std::inserter(variables, variables.begin()));
     return variables;
 }

src/libstore/machines.cc

@@ -6,7 +6,8 @@
 
 namespace nix {
 
-Machine::Machine(decltype(storeUri) storeUri,
+Machine::Machine(
+    const std::string & storeUri,
     decltype(systemTypes) systemTypes,
     decltype(sshKey) sshKey,
     decltype(maxJobs) maxJobs,
@@ -14,7 +15,7 @@ Machine::Machine(decltype(storeUri) storeUri,
     decltype(supportedFeatures) supportedFeatures,
     decltype(mandatoryFeatures) mandatoryFeatures,
     decltype(sshPublicHostKey) sshPublicHostKey) :
-    storeUri(
+    storeUri(StoreReference::parse(
         // Backwards compatibility: if the URI is schemeless, is not a path,
         // and is not one of the special store connection words, prepend
         // ssh://.
@@ -28,7 +29,7 @@ Machine::Machine(decltype(storeUri) storeUri,
         || hasPrefix(storeUri, "local?")
         || hasPrefix(storeUri, "?")
         ? storeUri
-        : "ssh://" + storeUri),
+        : "ssh://" + storeUri)),
     systemTypes(systemTypes),
     sshKey(sshKey),
     maxJobs(maxJobs),
@@ -63,23 +64,26 @@ bool Machine::mandatoryMet(const std::set<std::string> & features) const
         });
 }
 
-ref<Store> Machine::openStore() const
+StoreReference Machine::completeStoreReference() const
 {
-    Store::Params storeParams;
+    auto storeUri = this->storeUri;
-    if (hasPrefix(storeUri, "ssh://")) {
+
-        storeParams["max-connections"] = "1";
+    auto * generic = std::get_if<StoreReference::Specified>(&storeUri.variant);
-        storeParams["log-fd"] = "4";
+
+    if (generic && generic->scheme == "ssh") {
+        storeUri.params["max-connections"] = "1";
+        storeUri.params["log-fd"] = "4";
     }
 
-    if (hasPrefix(storeUri, "ssh://") || hasPrefix(storeUri, "ssh-ng://")) {
+    if (generic && (generic->scheme == "ssh" || generic->scheme == "ssh-ng")) {
         if (sshKey != "")
-            storeParams["ssh-key"] = sshKey;
+            storeUri.params["ssh-key"] = sshKey;
         if (sshPublicHostKey != "")
-            storeParams["base64-ssh-public-host-key"] = sshPublicHostKey;
+            storeUri.params["base64-ssh-public-host-key"] = sshPublicHostKey;
     }
 
     {
-        auto & fs = storeParams["system-features"];
+        auto & fs = storeUri.params["system-features"];
         auto append = [&](auto feats) {
             for (auto & f : feats) {
                 if (fs.size() > 0) fs += ' ';
@@ -90,7 +94,12 @@ ref<Store> Machine::openStore() const
         append(mandatoryFeatures);
     }
 
-    return nix::openStore(storeUri, storeParams);
+    return storeUri;
+}
+
+ref<Store> Machine::openStore() const
+{
+    return nix::openStore(completeStoreReference());
 }
 
 static std::vector<std::string> expandBuilderLines(const std::string & builders)

src/libstore/machines.hh

@@ -2,6 +2,7 @@
 ///@file
 
 #include "types.hh"
+#include "store-reference.hh"
 
 namespace nix {
 
@@ -9,7 +10,7 @@ class Store;
 
 struct Machine {
 
-    const std::string storeUri;
+    const StoreReference storeUri;
     const std::set<std::string> systemTypes;
     const std::string sshKey;
     const unsigned int maxJobs;
@@ -36,7 +37,8 @@ struct Machine {
      */
     bool mandatoryMet(const std::set<std::string> & features) const;
 
-    Machine(decltype(storeUri) storeUri,
+    Machine(
+        const std::string & storeUri,
         decltype(systemTypes) systemTypes,
         decltype(sshKey) sshKey,
         decltype(maxJobs) maxJobs,
@@ -45,6 +47,21 @@ struct Machine {
         decltype(mandatoryFeatures) mandatoryFeatures,
         decltype(sshPublicHostKey) sshPublicHostKey);
 
+    /**
+     * Elaborate `storeUri` into a complete store reference,
+     * incorporating information from the other fields of the `Machine`
+     * as applicable.
+     */
+    StoreReference completeStoreReference() const;
+
+    /**
+     * Open a `Store` for this machine.
+     *
+     * Just a simple function composition:
+     * ```c++
+     * nix::openStore(completeStoreReference())
+     * ```
+     */
     ref<Store> openStore() const;
 };
 

src/libstore/store-api.cc

@@ -8,7 +8,6 @@
 #include "util.hh"
 #include "nar-info-disk-cache.hh"
 #include "thread-pool.hh"
-#include "url.hh"
 #include "references.hh"
 #include "archive.hh"
 #include "callback.hh"
@@ -1267,109 +1266,63 @@ Derivation Store::readInvalidDerivation(const StorePath & drvPath)
 
 namespace nix {
 
-/* Split URI into protocol+hierarchy part and its parameter set. */
+ref<Store> openStore(const std::string & uri,
-std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_)
-{
-    auto uri(uri_);
-    Store::Params params;
-    auto q = uri.find('?');
-    if (q != std::string::npos) {
-        params = decodeQuery(uri.substr(q + 1));
-        uri = uri_.substr(0, q);
-    }
-    return {uri, params};
-}
-
-static bool isNonUriPath(const std::string & spec)
-{
-    return
-        // is not a URL
-        spec.find("://") == std::string::npos
-        // Has at least one path separator, and so isn't a single word that
-        // might be special like "auto"
-        && spec.find("/") != std::string::npos;
-}
-
-std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Params & params)
-{
-    // TODO reenable on Windows once we have `LocalStore` and
-    // `UDSRemoteStore`.
-    if (uri == "" || uri == "auto") {
-        auto stateDir = getOr(params, "state", settings.nixStateDir);
-        if (access(stateDir.c_str(), R_OK | W_OK) == 0)
-            return std::make_shared<LocalStore>(params);
-        else if (pathExists(settings.nixDaemonSocketFile))
-            return std::make_shared<UDSRemoteStore>(params);
-        #if __linux__
-        else if (!pathExists(stateDir)
-            && params.empty()
-            && !isRootUser()
-            && !getEnv("NIX_STORE_DIR").has_value()
-            && !getEnv("NIX_STATE_DIR").has_value())
-        {
-            /* If /nix doesn't exist, there is no daemon socket, and
-               we're not root, then automatically set up a chroot
-               store in ~/.local/share/nix/root. */
-            auto chrootStore = getDataDir() + "/nix/root";
-            if (!pathExists(chrootStore)) {
-                try {
-                    createDirs(chrootStore);
-                } catch (Error & e) {
-                    return std::make_shared<LocalStore>(params);
-                }
-                warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
-            } else
-                debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
-            return std::make_shared<LocalStore>("local", chrootStore, params);
-        }
-        #endif
-        else
-            return std::make_shared<LocalStore>(params);
-    } else if (uri == "daemon") {
-        return std::make_shared<UDSRemoteStore>(params);
-    } else if (uri == "local") {
-        return std::make_shared<LocalStore>(params);
-    } else if (isNonUriPath(uri)) {
-        return std::make_shared<LocalStore>("local", absPath(uri), params);
-    } else {
-        return nullptr;
-    }
-}
-
-ref<Store> openStore(const std::string & uri_,
     const Store::Params & extraParams)
 {
-    auto params = extraParams;
+    return openStore(StoreReference::parse(uri, extraParams));
-    try {
+}
-        auto parsedUri = parseURL(uri_);
-        params.insert(parsedUri.query.begin(), parsedUri.query.end());
 
-        auto baseURI = parsedUri.authority.value_or("") + parsedUri.path;
+ref<Store> openStore(StoreReference && storeURI)
+{
+    auto & params = storeURI.params;
 
-        for (auto implem : *Implementations::registered) {
+    auto store = std::visit(overloaded {
-            if (implem.uriSchemes.count(parsedUri.scheme)) {
+        [&](const StoreReference::Auto &) -> std::shared_ptr<Store> {
-                auto store = implem.create(parsedUri.scheme, baseURI, params);
+            auto stateDir = getOr(params, "state", settings.nixStateDir);
-                if (store) {
+            if (access(stateDir.c_str(), R_OK | W_OK) == 0)
-                    experimentalFeatureSettings.require(store->experimentalFeature());
+                return std::make_shared<LocalStore>(params);
-                    store->init();
+            else if (pathExists(settings.nixDaemonSocketFile))
-                    store->warnUnknownSettings();
+                return std::make_shared<UDSRemoteStore>(params);
-                    return ref<Store>(store);
+            #if __linux__
-                }
+            else if (!pathExists(stateDir)
+                && params.empty()
+                && !isRootUser()
+                && !getEnv("NIX_STORE_DIR").has_value()
+                && !getEnv("NIX_STATE_DIR").has_value())
+            {
+                /* If /nix doesn't exist, there is no daemon socket, and
+                   we're not root, then automatically set up a chroot
+                   store in ~/.local/share/nix/root. */
+                auto chrootStore = getDataDir() + "/nix/root";
+                if (!pathExists(chrootStore)) {
+                    try {
+                        createDirs(chrootStore);
+                    } catch (Error & e) {
+                        return std::make_shared<LocalStore>(params);
+                    }
+                    warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
+                } else
+                    debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
+                return std::make_shared<LocalStore>("local", chrootStore, params);
             }
-        }
+            #endif
-    }
+            else
-    catch (BadURL &) {
+                return std::make_shared<LocalStore>(params);
-        auto [uri, uriParams] = splitUriAndParams(uri_);
+        },
-        params.insert(uriParams.begin(), uriParams.end());
+        [&](const StoreReference::Specified & g) {
+            for (auto implem : *Implementations::registered)
+                if (implem.uriSchemes.count(g.scheme))
+                    return implem.create(g.scheme, g.authority, params);
 
-        if (auto store = openFromNonUri(uri, params)) {
+            throw Error("don't know how to open Nix store with scheme '%s'", g.scheme);
-            experimentalFeatureSettings.require(store->experimentalFeature());
+        },
-            store->warnUnknownSettings();
+    }, storeURI.variant);
-            return ref<Store>(store);
-        }
-    }
 
-    throw Error("don't know how to open Nix store '%s'", uri_);
+    experimentalFeatureSettings.require(store->experimentalFeature());
+    store->warnUnknownSettings();
+    store->init();
+
+    return ref<Store> { store };
 }
 
 std::list<ref<Store>> getDefaultSubstituters()

src/libstore/store-api.hh

@@ -13,6 +13,7 @@
 #include "path-info.hh"
 #include "repair-flag.hh"
 #include "store-dir-config.hh"
+#include "store-reference.hh"
 #include "source-path.hh"
 
 #include <nlohmann/json_fwd.hpp>
@@ -65,7 +66,7 @@ MakeError(Unsupported, Error);
 MakeError(SubstituteGone, Error);
 MakeError(SubstituterDisabled, Error);
 
-MakeError(InvalidStoreURI, Error);
+MakeError(InvalidStoreReference, Error);
 
 struct Realisation;
 struct RealisedPath;
@@ -102,7 +103,7 @@ typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
 
 struct StoreConfig : public StoreDirConfig
 {
-    typedef std::map<std::string, std::string> Params;
+    using Params = StoreReference::Params;
 
     using StoreDirConfig::StoreDirConfig;
 
@@ -859,34 +860,13 @@ OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * ev
 /**
  * @return a Store object to access the Nix store denoted by
  * ‘uri’ (slight misnomer...).
- *
+ */
- * @param uri Supported values are:
+ref<Store> openStore(StoreReference && storeURI);
- *
+
- * - ‘local’: The Nix store in /nix/store and database in
+
- *   /nix/var/nix/db, accessed directly.
+/**
- *
+ * Opens the store at `uri`, where `uri` is in the format expected by `StoreReference::parse`
- * - ‘daemon’: The Nix store accessed via a Unix domain socket
+
- *   connection to nix-daemon.
- *
- * - ‘unix://<path>’: The Nix store accessed via a Unix domain socket
- *   connection to nix-daemon, with the socket located at <path>.
- *
- * - ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on
- *   whether the user has write access to the local Nix
- *   store/database.
- *
- * - ‘file://<path>’: A binary cache stored in <path>.
- *
- * - ‘https://<path>’: A binary cache accessed via HTTP.
- *
- * - ‘s3://<path>’: A writable binary cache stored on Amazon's Simple
- *   Storage Service.
- *
- * - ‘ssh://[user@]<host>’: A remote Nix store accessed by running
- *   ‘nix-store --serve’ via SSH.
- *
- * You can pass parameters to the store type by appending
- * ‘?key=value&key=value&...’ to the URI.
  */
 ref<Store> openStore(const std::string & uri = settings.storeUri.get(),
     const Store::Params & extraParams = Store::Params());
@@ -957,11 +937,6 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
     std::istream & str,
     std::optional<HashResult> hashGiven = std::nullopt);
 
-/**
- * Split URI into protocol+hierarchy part and its parameter set.
- */
-std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
-
 const ContentAddress * getDerivationCA(const BasicDerivation & drv);
 
 std::map<DrvOutput, StorePath> drvOutputReferences(

src/libstore/store-reference.cc

@@ -0,0 +1,116 @@
+#include <regex>
+
+#include "error.hh"
+#include "url.hh"
+#include "store-reference.hh"
+#include "file-system.hh"
+#include "util.hh"
+
+namespace nix {
+
+static bool isNonUriPath(const std::string & spec)
+{
+    return
+        // is not a URL
+        spec.find("://") == std::string::npos
+        // Has at least one path separator, and so isn't a single word that
+        // might be special like "auto"
+        && spec.find("/") != std::string::npos;
+}
+
+std::string StoreReference::render() const
+{
+    std::string res;
+
+    std::visit(
+        overloaded{
+            [&](const StoreReference::Auto &) { res = "auto"; },
+            [&](const StoreReference::Specified & g) {
+                res = g.scheme;
+                res += "://";
+                res += g.authority;
+            },
+        },
+        variant);
+
+    if (!params.empty()) {
+        res += "?";
+        res += encodeQuery(params);
+    }
+
+    return res;
+}
+
+StoreReference StoreReference::parse(const std::string & uri, const StoreReference::Params & extraParams)
+{
+    auto params = extraParams;
+    try {
+        auto parsedUri = parseURL(uri);
+        params.insert(parsedUri.query.begin(), parsedUri.query.end());
+
+        auto baseURI = parsedUri.authority.value_or("") + parsedUri.path;
+
+        return {
+            .variant =
+                Specified{
+                    .scheme = std::move(parsedUri.scheme),
+                    .authority = std::move(baseURI),
+                },
+            .params = std::move(params),
+        };
+    } catch (BadURL &) {
+        auto [baseURI, uriParams] = splitUriAndParams(uri);
+        params.insert(uriParams.begin(), uriParams.end());
+
+        if (baseURI == "" || baseURI == "auto") {
+            return {
+                .variant = Auto{},
+                .params = std::move(params),
+            };
+        } else if (baseURI == "daemon") {
+            return {
+                .variant =
+                    Specified{
+                        .scheme = "unix",
+                        .authority = "",
+                    },
+                .params = std::move(params),
+            };
+        } else if (baseURI == "local") {
+            return {
+                .variant =
+                    Specified{
+                        .scheme = "local",
+                        .authority = "",
+                    },
+                .params = std::move(params),
+            };
+        } else if (isNonUriPath(baseURI)) {
+            return {
+                .variant =
+                    Specified{
+                        .scheme = "local",
+                        .authority = absPath(baseURI),
+                    },
+                .params = std::move(params),
+            };
+        }
+    }
+
+    throw UsageError("Cannot parse Nix store '%s'", uri);
+}
+
+/* Split URI into protocol+hierarchy part and its parameter set. */
+std::pair<std::string, StoreReference::Params> splitUriAndParams(const std::string & uri_)
+{
+    auto uri(uri_);
+    StoreReference::Params params;
+    auto q = uri.find('?');
+    if (q != std::string::npos) {
+        params = decodeQuery(uri.substr(q + 1));
+        uri = uri_.substr(0, q);
+    }
+    return {uri, params};
+}
+
+}

src/libstore/store-reference.hh

@@ -0,0 +1,92 @@
+#pragma once
+///@file
+
+#include <variant>
+
+#include "types.hh"
+
+namespace nix {
+
+/**
+ * A parsed Store URI (URI is a slight misnomer...), parsed but not yet
+ * resolved to a specific instance and query parms validated.
+ *
+ * Supported values are:
+ *
+ * - ‘local’: The Nix store in /nix/store and database in
+ *   /nix/var/nix/db, accessed directly.
+ *
+ * - ‘daemon’: The Nix store accessed via a Unix domain socket
+ *   connection to nix-daemon.
+ *
+ * - ‘unix://<path>’: The Nix store accessed via a Unix domain socket
+ *   connection to nix-daemon, with the socket located at <path>.
+ *
+ * - ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on
+ *   whether the user has write access to the local Nix
+ *   store/database.
+ *
+ * - ‘file://<path>’: A binary cache stored in <path>.
+ *
+ * - ‘https://<path>’: A binary cache accessed via HTTP.
+ *
+ * - ‘s3://<path>’: A writable binary cache stored on Amazon's Simple
+ *   Storage Service.
+ *
+ * - ‘ssh://[user@]<host>’: A remote Nix store accessed by running
+ *   ‘nix-store --serve’ via SSH.
+ *
+ * You can pass parameters to the store type by appending
+ * ‘?key=value&key=value&...’ to the URI.
+ */
+struct StoreReference
+{
+    using Params = std::map<std::string, std::string>;
+
+    /**
+     * Special store reference `""` or `"auto"`
+     */
+    struct Auto
+    {
+        inline bool operator==(const Auto & rhs) const = default;
+        inline auto operator<=>(const Auto & rhs) const = default;
+    };
+
+    /**
+     * General case, a regular `scheme://authority` URL.
+     */
+    struct Specified
+    {
+        std::string scheme;
+        std::string authority = "";
+
+        bool operator==(const Specified & rhs) const = default;
+        auto operator<=>(const Specified & rhs) const = default;
+    };
+
+    typedef std::variant<Auto, Specified> Variant;
+
+    Variant variant;
+
+    Params params;
+
+    bool operator==(const StoreReference & rhs) const = default;
+    auto operator<=>(const StoreReference & rhs) const = default;
+
+    /**
+     * Render the whole store reference as a URI, including parameters.
+     */
+    std::string render() const;
+
+    /**
+     * Parse a URI into a store reference.
+     */
+    static StoreReference parse(const std::string & uri, const Params & extraParams = Params{});
+};
+
+/**
+ * Split URI into protocol+hierarchy part and its parameter set.
+ */
+std::pair<std::string, StoreReference::Params> splitUriAndParams(const std::string & uri);
+
+}

src/libstore/windows/pathlocks.cc

@@ -35,8 +35,13 @@ void PathLocks::unlock()
 AutoCloseFD openLockFile(const Path & path, bool create)
 {
     AutoCloseFD desc = CreateFileA(
-        path.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL,
+        path.c_str(),
-        create ? OPEN_ALWAYS : OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_POSIX_SEMANTICS, NULL);
+        GENERIC_READ | GENERIC_WRITE,
+        FILE_SHARE_READ | FILE_SHARE_WRITE,
+        NULL,
+        create ? OPEN_ALWAYS : OPEN_EXISTING,
+        FILE_ATTRIBUTE_NORMAL | FILE_FLAG_POSIX_SEMANTICS,
+        NULL);
     if (desc.get() == INVALID_HANDLE_VALUE)
         warn("%s: %s", path, std::to_string(GetLastError()));
 

src/libutil/compression.cc

@@ -263,8 +263,13 @@ struct BrotliCompressionSink : ChunkedCompressionSink
             checkInterrupt();
 
             if (!BrotliEncoderCompressStream(
-                    state, data.data() ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH, &avail_in, &next_in,
+                    state,
-                    &avail_out, &next_out, nullptr))
+                    data.data() ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH,
+                    &avail_in,
+                    &next_in,
+                    &avail_out,
+                    &next_out,
+                    nullptr))
                 throw CompressionError("error while compressing brotli compression");
 
             if (avail_out < sizeof(outbuf) || avail_in == 0) {
@@ -280,8 +285,8 @@ struct BrotliCompressionSink : ChunkedCompressionSink
 
 ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel, int level)
 {
-    std::vector<std::string> la_supports = {"bzip2", "compress", "grzip", "gzip", "lrzip", "lz4",
+    std::vector<std::string> la_supports = {
-                                            "lzip",  "lzma",     "lzop",  "xz",   "zstd"};
+        "bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd"};
     if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) {
         return make_ref<ArchiveCompressionSink>(nextSink, method, parallel, level);
     }

src/libutil/url.hh

@@ -33,6 +33,8 @@ std::string percentEncode(std::string_view s, std::string_view keep="");
 
 std::map<std::string, std::string> decodeQuery(const std::string & query);
 
+std::string encodeQuery(const std::map<std::string, std::string> & query);
+
 ParsedURL parseURL(const std::string & url);
 
 /**

src/libutil/windows/file-system.cc

@@ -5,8 +5,13 @@ namespace nix {
 Descriptor openDirectory(const std::filesystem::path & path)
 {
     return CreateFileW(
-        path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, OPEN_EXISTING,
+        path.c_str(),
-        FILE_FLAG_BACKUP_SEMANTICS, NULL);
+        GENERIC_READ,
+        FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+        NULL,
+        OPEN_EXISTING,
+        FILE_FLAG_BACKUP_SEMANTICS,
+        NULL);
 }
 
 }

tests/unit/libexpr/nix_api_value.cc

@@ -256,10 +256,13 @@ TEST_F(nix_api_expr_test, nix_value_init)
 
     Value * f = nix_alloc_value(ctx, state);
     nix_expr_eval_from_string(
-        ctx, state, R"(
+        ctx,
+        state,
+        R"(
         a: a * a
     )",
-        "<test>", f);
+        "<test>",
+        f);
 
     // Test
 
@@ -325,20 +328,26 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg)
 
     Value * f = nix_alloc_value(ctx, state);
     nix_expr_eval_from_string(
-        ctx, state, R"(
+        ctx,
+        state,
+        R"(
         a: { foo = a; }
     )",
-        "<test>", f);
+        "<test>",
+        f);
     assert_ctx_ok();
 
     Value * e = nix_alloc_value(ctx, state);
     {
         Value * g = nix_alloc_value(ctx, state);
         nix_expr_eval_from_string(
-            ctx, state, R"(
+            ctx,
+            state,
+            R"(
             _ignore: throw "error message for test case nix_value_init_apply_lazy_arg"
         )",
-            "<test>", g);
+            "<test>",
+            g);
         assert_ctx_ok();
 
         nix_init_apply(ctx, e, g, g);

tests/unit/libstore/data/store-reference/auto.txt

@@ -0,0 +1 @@
+auto

tests/unit/libstore/data/store-reference/auto_param.txt

@@ -0,0 +1 @@
+auto?root=/foo/bar/baz

tests/unit/libstore/data/store-reference/local_1.txt

@@ -0,0 +1 @@
+local://?root=/foo/bar/baz

tests/unit/libstore/data/store-reference/local_2.txt

@@ -0,0 +1 @@
+local:///foo/bar/baz?trusted=true

tests/unit/libstore/data/store-reference/local_shorthand_1.txt

@@ -0,0 +1 @@
+local?root=/foo/bar/baz

tests/unit/libstore/data/store-reference/local_shorthand_2.txt

@@ -0,0 +1 @@
+/foo/bar/baz?trusted=true

tests/unit/libstore/data/store-reference/ssh.txt

@@ -0,0 +1 @@
+ssh://localhost

tests/unit/libstore/data/store-reference/unix.txt

@@ -0,0 +1 @@
+unix://?max-connections=7&trusted=true

tests/unit/libstore/data/store-reference/unix_shorthand.txt

@@ -0,0 +1 @@
+daemon?max-connections=7&trusted=true

tests/unit/libstore/machines.cc

@@ -3,24 +3,16 @@
 #include "file-system.hh"
 #include "util.hh"
 
+#include <gtest/gtest.h>
 #include <gmock/gmock-matchers.h>
 
 using testing::Contains;
 using testing::ElementsAre;
-using testing::EndsWith;
 using testing::Eq;
 using testing::Field;
 using testing::SizeIs;
 
-using nix::absPath;
+using namespace nix;
-using nix::FormatError;
-using nix::UsageError;
-using nix::getMachines;
-using nix::Machine;
-using nix::Machines;
-using nix::pathExists;
-using nix::Settings;
-using nix::settings;
 
 class Environment : public ::testing::Environment {
   public:
@@ -40,7 +32,7 @@ TEST(machines, getMachinesUriOnly) {
     settings.builders = "nix@scratchy.labs.cs.uu.nl";
     Machines actual = getMachines();
     ASSERT_THAT(actual, SizeIs(1));
-    EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl")));
+    EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl"))));
     EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS")));
     EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0)));
     EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1)));
@@ -54,7 +46,7 @@ TEST(machines, getMachinesDefaults) {
     settings.builders = "nix@scratchy.labs.cs.uu.nl - - - - - - -";
     Machines actual = getMachines();
     ASSERT_THAT(actual, SizeIs(1));
-    EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl")));
+    EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl"))));
     EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS")));
     EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0)));
     EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1)));
@@ -64,20 +56,31 @@ TEST(machines, getMachinesDefaults) {
     EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, SizeIs(0)));
 }
 
+MATCHER_P(AuthorityMatches, authority, "") {
+    *result_listener
+        << "where the authority of "
+        << arg.render()
+        << " is "
+        << authority;
+    auto * generic = std::get_if<StoreReference::Specified>(&arg.variant);
+    if (!generic) return false;
+    return generic->authority == authority;
+}
+
 TEST(machines, getMachinesWithNewLineSeparator) {
     settings.builders = "nix@scratchy.labs.cs.uu.nl\nnix@itchy.labs.cs.uu.nl";
     Machines actual = getMachines();
     ASSERT_THAT(actual, SizeIs(2));
-    EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))));
+    EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))));
-    EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl"))));
+    EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl"))));
 }
 
 TEST(machines, getMachinesWithSemicolonSeparator) {
     settings.builders = "nix@scratchy.labs.cs.uu.nl ; nix@itchy.labs.cs.uu.nl";
     Machines actual = getMachines();
     EXPECT_THAT(actual, SizeIs(2));
-    EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))));
+    EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))));
-    EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl"))));
+    EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl"))));
 }
 
 TEST(machines, getMachinesWithCorrectCompleteSingleBuilder) {
@@ -86,7 +89,7 @@ TEST(machines, getMachinesWithCorrectCompleteSingleBuilder) {
                         "benchmark SSH+HOST+PUBLIC+KEY+BASE64+ENCODED==";
     Machines actual = getMachines();
     ASSERT_THAT(actual, SizeIs(1));
-    EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")));
+    EXPECT_THAT(actual[0], Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")));
     EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux")));
     EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto")));
     EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8)));
@@ -104,7 +107,7 @@ TEST(machines,
         "KEY+BASE64+ENCODED==";
     Machines actual = getMachines();
     ASSERT_THAT(actual, SizeIs(1));
-    EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")));
+    EXPECT_THAT(actual[0], Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")));
     EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux")));
     EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto")));
     EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8)));
@@ -120,7 +123,7 @@ TEST(machines, getMachinesWithMultiOptions) {
                         "MandatoryFeature1,MandatoryFeature2";
     Machines actual = getMachines();
     ASSERT_THAT(actual, SizeIs(1));
-    EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")));
+    EXPECT_THAT(actual[0], Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")));
     EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("Arch1", "Arch2")));
     EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("SupportedFeature1", "SupportedFeature2")));
     EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("MandatoryFeature1", "MandatoryFeature2")));
@@ -146,9 +149,9 @@ TEST(machines, getMachinesWithCorrectFileReference) {
     settings.builders = std::string("@") + path;
     Machines actual = getMachines();
     ASSERT_THAT(actual, SizeIs(3));
-    EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))));
+    EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))));
-    EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl"))));
+    EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl"))));
-    EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@poochie.labs.cs.uu.nl"))));
+    EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@poochie.labs.cs.uu.nl"))));
 }
 
 TEST(machines, getMachinesWithCorrectFileReferenceToEmptyFile) {

tests/unit/libstore/store-reference.cc

@@ -0,0 +1,123 @@
+#include <nlohmann/json.hpp>
+#include <gtest/gtest.h>
+
+#include "file-system.hh"
+#include "store-reference.hh"
+
+#include "tests/characterization.hh"
+#include "tests/libstore.hh"
+
+namespace nix {
+
+using nlohmann::json;
+
+class StoreReferenceTest : public CharacterizationTest, public LibStoreTest
+{
+    Path unitTestData = getUnitTestData() + "/store-reference";
+
+    Path goldenMaster(PathView testStem) const override
+    {
+        return unitTestData + "/" + testStem + ".txt";
+    }
+};
+
+#define URI_TEST_READ(STEM, OBJ)                         \
+  TEST_F(StoreReferenceTest, PathInfo_##STEM##_from_uri) \
+  {                                                      \
+    readTest(#STEM, ([&](const auto & encoded) {         \
+           StoreReference expected = OBJ;                \
+           auto got = StoreReference::parse(encoded);    \
+           ASSERT_EQ(got, expected);                     \
+             }));                                        \
+  }
+
+#define URI_TEST_WRITE(STEM, OBJ)                                                           \
+  TEST_F(StoreReferenceTest, PathInfo_##STEM##_to_uri)                                      \
+  {                                                                                         \
+    writeTest(                                                                              \
+        #STEM,                                                                              \
+        [&]() -> StoreReference { return OBJ; },                                            \
+        [](const auto & file) { return StoreReference::parse(readFile(file)); },            \
+        [](const auto & file, const auto & got) { return writeFile(file, got.render()); }); \
+  }
+
+#define URI_TEST(STEM, OBJ) \
+  URI_TEST_READ(STEM, OBJ)  \
+  URI_TEST_WRITE(STEM, OBJ)
+
+URI_TEST(
+    auto,
+    (StoreReference{
+        .variant = StoreReference::Auto{},
+        .params = {},
+    }))
+
+URI_TEST(
+    auto_param,
+    (StoreReference{
+        .variant = StoreReference::Auto{},
+        .params =
+            {
+                {"root", "/foo/bar/baz"},
+            },
+    }))
+
+static StoreReference localExample_1{
+    .variant =
+        StoreReference::Specified{
+            .scheme = "local",
+        },
+    .params =
+        {
+            {"root", "/foo/bar/baz"},
+        },
+};
+
+static StoreReference localExample_2{
+    .variant =
+        StoreReference::Specified{
+            .scheme = "local",
+            .authority = "/foo/bar/baz",
+        },
+    .params =
+        {
+            {"trusted", "true"},
+        },
+};
+
+URI_TEST(local_1, localExample_1)
+
+URI_TEST(local_2, localExample_2)
+
+URI_TEST_READ(local_shorthand_1, localExample_1)
+
+URI_TEST_READ(local_shorthand_2, localExample_2)
+
+static StoreReference unixExample{
+    .variant =
+        StoreReference::Specified{
+            .scheme = "unix",
+        },
+    .params =
+        {
+            {"max-connections", "7"},
+            {"trusted", "true"},
+        },
+};
+
+URI_TEST(unix, unixExample)
+
+URI_TEST_READ(unix_shorthand, unixExample)
+
+URI_TEST(
+    ssh,
+    (StoreReference{
+        .variant =
+            StoreReference::Specified{
+                .scheme = "ssh",
+                .authority = "localhost",
+            },
+        .params = {},
+    }))
+
+}