From b3ebcc5aad1f3c8f56895efc0632af47a5dca532 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 23 Jan 2024 15:36:03 -0500 Subject: [PATCH] Use the new `StoreReference` in `Machine` This makes the remote builder abstract syntax more robust. --- src/build-remote/build-remote.cc | 22 +++++++-------- src/libstore/machines.cc | 35 +++++++++++++++--------- src/libstore/machines.hh | 21 ++++++++++++-- tests/unit/libstore/machines.cc | 47 +++++++++++++++++--------------- 4 files changed, 77 insertions(+), 48 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 18eee830b..582e6d623 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -37,7 +37,7 @@ static std::string currentLoad; static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) { - return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); + return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true); } static bool allSupportedLocally(Store & store, const std::set& requiredFeatures) { @@ -99,7 +99,7 @@ static int main_build_remote(int argc, char * * argv) } std::optional drvPath; - std::string storeUri; + StoreReference storeUri; while (true) { @@ -135,7 +135,7 @@ static int main_build_remote(int argc, char * * argv) Machine * bestMachine = nullptr; uint64_t bestLoad = 0; for (auto & m : machines) { - debug("considering building on remote machine '%s'", m.storeUri); + debug("considering building on remote machine '%s'", m.storeUri.render()); if (m.enabled && m.systemSupported(neededSystem) && @@ -233,7 +233,7 @@ static int main_build_remote(int argc, char * * argv) try { - Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri.render())); sshStore = bestMachine->openStore(); sshStore->connect(); @@ -242,7 +242,7 @@ static int main_build_remote(int argc, char * * argv) } catch (std::exception & e) { auto msg = chomp(drainFD(5, false)); printError("cannot build on '%s': %s%s", - bestMachine->storeUri, e.what(), + bestMachine->storeUri.render(), e.what(), msg.empty() ? "" : ": " + msg); bestMachine->enabled = false; continue; @@ -257,15 +257,15 @@ connected: assert(sshStore); - std::cerr << "# accept\n" << storeUri << "\n"; + std::cerr << "# accept\n" << storeUri.render() << "\n"; auto inputs = readStrings(source); auto wantedOutputs = readStrings(source); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); + AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri.render()) + ".upload-lock", true); { - Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri.render())); auto old = signal(SIGALRM, handleAlarm); alarm(15 * 60); @@ -278,7 +278,7 @@ connected: auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute; { - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri.render())); copyPaths(*store, *sshStore, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute); } @@ -316,7 +316,7 @@ connected: optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv); auto & result = *optResult; if (!result.success()) - throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); + throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri.render(), result.errorMsg); } else { copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute); auto res = sshStore->buildPathsWithResults({ @@ -359,7 +359,7 @@ connected: } if (!missingPaths.empty()) { - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri.render())); if (auto localStore = store.dynamic_pointer_cast()) for (auto & path : missingPaths) localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */ diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 2d461c63a..644762242 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -6,7 +6,8 @@ namespace nix { -Machine::Machine(decltype(storeUri) storeUri, +Machine::Machine( + const std::string & storeUri, decltype(systemTypes) systemTypes, decltype(sshKey) sshKey, decltype(maxJobs) maxJobs, @@ -14,7 +15,7 @@ Machine::Machine(decltype(storeUri) storeUri, decltype(supportedFeatures) supportedFeatures, decltype(mandatoryFeatures) mandatoryFeatures, decltype(sshPublicHostKey) sshPublicHostKey) : - storeUri( + storeUri(StoreReference::parse( // Backwards compatibility: if the URI is schemeless, is not a path, // and is not one of the special store connection words, prepend // ssh://. @@ -28,7 +29,7 @@ Machine::Machine(decltype(storeUri) storeUri, || hasPrefix(storeUri, "local?") || hasPrefix(storeUri, "?") ? storeUri - : "ssh://" + storeUri), + : "ssh://" + storeUri)), systemTypes(systemTypes), sshKey(sshKey), maxJobs(maxJobs), @@ -63,23 +64,26 @@ bool Machine::mandatoryMet(const std::set & features) const }); } -ref Machine::openStore() const +StoreReference Machine::completeStoreReference() const { - Store::Params storeParams; - if (hasPrefix(storeUri, "ssh://")) { - storeParams["max-connections"] = "1"; - storeParams["log-fd"] = "4"; + auto storeUri = this->storeUri; + + auto * generic = std::get_if(&storeUri.variant); + + if (generic && generic->scheme == "ssh") { + storeUri.params["max-connections"] = "1"; + storeUri.params["log-fd"] = "4"; } - if (hasPrefix(storeUri, "ssh://") || hasPrefix(storeUri, "ssh-ng://")) { + if (generic && (generic->scheme == "ssh" || generic->scheme == "ssh-ng")) { if (sshKey != "") - storeParams["ssh-key"] = sshKey; + storeUri.params["ssh-key"] = sshKey; if (sshPublicHostKey != "") - storeParams["base64-ssh-public-host-key"] = sshPublicHostKey; + storeUri.params["base64-ssh-public-host-key"] = sshPublicHostKey; } { - auto & fs = storeParams["system-features"]; + auto & fs = storeUri.params["system-features"]; auto append = [&](auto feats) { for (auto & f : feats) { if (fs.size() > 0) fs += ' '; @@ -90,7 +94,12 @@ ref Machine::openStore() const append(mandatoryFeatures); } - return nix::openStore(storeUri, storeParams); + return storeUri; +} + +ref Machine::openStore() const +{ + return nix::openStore(completeStoreReference()); } static std::vector expandBuilderLines(const std::string & builders) diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh index 8516409d4..97980df8d 100644 --- a/src/libstore/machines.hh +++ b/src/libstore/machines.hh @@ -2,6 +2,7 @@ ///@file #include "types.hh" +#include "store-reference.hh" namespace nix { @@ -9,7 +10,7 @@ class Store; struct Machine { - const std::string storeUri; + const StoreReference storeUri; const std::set systemTypes; const std::string sshKey; const unsigned int maxJobs; @@ -36,7 +37,8 @@ struct Machine { */ bool mandatoryMet(const std::set & features) const; - Machine(decltype(storeUri) storeUri, + Machine( + const std::string & storeUri, decltype(systemTypes) systemTypes, decltype(sshKey) sshKey, decltype(maxJobs) maxJobs, @@ -45,6 +47,21 @@ struct Machine { decltype(mandatoryFeatures) mandatoryFeatures, decltype(sshPublicHostKey) sshPublicHostKey); + /** + * Elaborate `storeUri` into a complete store reference, + * incorporating information from the other fields of the `Machine` + * as applicable. + */ + StoreReference completeStoreReference() const; + + /** + * Open a `Store` for this machine. + * + * Just a simple function composition: + * ```c++ + * nix::openStore(completeStoreReference()) + * ``` + */ ref openStore() const; }; diff --git a/tests/unit/libstore/machines.cc b/tests/unit/libstore/machines.cc index 9fd7fda54..4107ba655 100644 --- a/tests/unit/libstore/machines.cc +++ b/tests/unit/libstore/machines.cc @@ -3,24 +3,16 @@ #include "file-system.hh" #include "util.hh" +#include #include using testing::Contains; using testing::ElementsAre; -using testing::EndsWith; using testing::Eq; using testing::Field; using testing::SizeIs; -using nix::absPath; -using nix::FormatError; -using nix::UsageError; -using nix::getMachines; -using nix::Machine; -using nix::Machines; -using nix::pathExists; -using nix::Settings; -using nix::settings; +using namespace nix; class Environment : public ::testing::Environment { public: @@ -40,7 +32,7 @@ TEST(machines, getMachinesUriOnly) { settings.builders = "nix@scratchy.labs.cs.uu.nl"; Machines actual = getMachines(); ASSERT_THAT(actual, SizeIs(1)); - EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl")))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); @@ -54,7 +46,7 @@ TEST(machines, getMachinesDefaults) { settings.builders = "nix@scratchy.labs.cs.uu.nl - - - - - - -"; Machines actual = getMachines(); ASSERT_THAT(actual, SizeIs(1)); - EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl")))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); @@ -64,20 +56,31 @@ TEST(machines, getMachinesDefaults) { EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, SizeIs(0))); } +MATCHER_P(AuthorityMatches, authority, "") { + *result_listener + << "where the authority of " + << arg.render() + << " is " + << authority; + auto * generic = std::get_if(&arg.variant); + if (!generic) return false; + return generic->authority == authority; +} + TEST(machines, getMachinesWithNewLineSeparator) { settings.builders = "nix@scratchy.labs.cs.uu.nl\nnix@itchy.labs.cs.uu.nl"; Machines actual = getMachines(); ASSERT_THAT(actual, SizeIs(2)); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl")))); } TEST(machines, getMachinesWithSemicolonSeparator) { settings.builders = "nix@scratchy.labs.cs.uu.nl ; nix@itchy.labs.cs.uu.nl"; Machines actual = getMachines(); EXPECT_THAT(actual, SizeIs(2)); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl")))); } TEST(machines, getMachinesWithCorrectCompleteSingleBuilder) { @@ -86,7 +89,7 @@ TEST(machines, getMachinesWithCorrectCompleteSingleBuilder) { "benchmark SSH+HOST+PUBLIC+KEY+BASE64+ENCODED=="; Machines actual = getMachines(); ASSERT_THAT(actual, SizeIs(1)); - EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux"))); EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto"))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8))); @@ -104,7 +107,7 @@ TEST(machines, "KEY+BASE64+ENCODED=="; Machines actual = getMachines(); ASSERT_THAT(actual, SizeIs(1)); - EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux"))); EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto"))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8))); @@ -120,7 +123,7 @@ TEST(machines, getMachinesWithMultiOptions) { "MandatoryFeature1,MandatoryFeature2"; Machines actual = getMachines(); ASSERT_THAT(actual, SizeIs(1)); - EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("Arch1", "Arch2"))); EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("SupportedFeature1", "SupportedFeature2"))); EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("MandatoryFeature1", "MandatoryFeature2"))); @@ -146,9 +149,9 @@ TEST(machines, getMachinesWithCorrectFileReference) { settings.builders = std::string("@") + path; Machines actual = getMachines(); ASSERT_THAT(actual, SizeIs(3)); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@poochie.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@poochie.labs.cs.uu.nl")))); } TEST(machines, getMachinesWithCorrectFileReferenceToEmptyFile) {