nordic-dev.net/nix
nordic-dev.net/nix
2
0
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2024-11-22 06:42:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use the new StoreReference in Machine

Browse Source 
This makes the remote builder abstract syntax more robust.
This commit is contained in:
John Ericson 2024-01-23 15:36:03 -05:00
parent b59a7a14c4
commit b3ebcc5aad
4 changed files with 77 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/build-remote/build-remote.cc
View File

@ -37,7 +37,7 @@ static std::string currentLoad;
static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
{ {
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true);
} }
static bool allSupportedLocally(Store & store, const std::set<std::string>& requiredFeatures) { static bool allSupportedLocally(Store & store, const std::set<std::string>& requiredFeatures) {
@ -99,7 +99,7 @@ static int main_build_remote(int argc, char * * argv)
} }
std::optional<StorePath> drvPath; std::optional<StorePath> drvPath;
std::string storeUri; StoreReference storeUri;
while (true) { while (true) {
@ -135,7 +135,7 @@ static int main_build_remote(int argc, char * * argv)
Machine * bestMachine = nullptr; Machine * bestMachine = nullptr;
uint64_t bestLoad = 0; uint64_t bestLoad = 0;
for (auto & m : machines) { for (auto & m : machines) {
debug("considering building on remote machine '%s'", m.storeUri); debug("considering building on remote machine '%s'", m.storeUri.render());
if (m.enabled && if (m.enabled &&
m.systemSupported(neededSystem) && m.systemSupported(neededSystem) &&
@ -233,7 +233,7 @@ static int main_build_remote(int argc, char * * argv)
try { try {
Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri)); Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri.render()));
sshStore = bestMachine->openStore(); sshStore = bestMachine->openStore();
sshStore->connect(); sshStore->connect();
@ -242,7 +242,7 @@ static int main_build_remote(int argc, char * * argv)
} catch (std::exception & e) { } catch (std::exception & e) {
auto msg = chomp(drainFD(5, false)); auto msg = chomp(drainFD(5, false));
printError("cannot build on '%s': %s%s", printError("cannot build on '%s': %s%s",
bestMachine->storeUri, e.what(), bestMachine->storeUri.render(), e.what(),
msg.empty() ? "" : ": " + msg); msg.empty() ? "" : ": " + msg);
bestMachine->enabled = false; bestMachine->enabled = false;
continue; continue;
@ -257,15 +257,15 @@ connected:
assert(sshStore); assert(sshStore);
std::cerr << "# accept\n" << storeUri << "\n"; std::cerr << "# accept\n" << storeUri.render() << "\n";
auto inputs = readStrings<PathSet>(source); auto inputs = readStrings<PathSet>(source);
auto wantedOutputs = readStrings<StringSet>(source); auto wantedOutputs = readStrings<StringSet>(source);
AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri.render()) + ".upload-lock", true);
{ {
Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri.render()));
auto old = signal(SIGALRM, handleAlarm); auto old = signal(SIGALRM, handleAlarm);
alarm(15 * 60); alarm(15 * 60);
@ -278,7 +278,7 @@ connected:
auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute; auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute;
{ {
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri)); Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri.render()));
copyPaths(*store, *sshStore, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute); copyPaths(*store, *sshStore, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute);
} }
@ -316,7 +316,7 @@ connected:
optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv); optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv);
auto & result = *optResult; auto & result = *optResult;
if (!result.success()) if (!result.success())
throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri.render(), result.errorMsg);
} else { } else {
copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute); copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute);
auto res = sshStore->buildPathsWithResults({ auto res = sshStore->buildPathsWithResults({
@ -359,7 +359,7 @@ connected:
} }
if (!missingPaths.empty()) { if (!missingPaths.empty()) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri.render()));
if (auto localStore = store.dynamic_pointer_cast<LocalStore>()) if (auto localStore = store.dynamic_pointer_cast<LocalStore>())
for (auto & path : missingPaths) for (auto & path : missingPaths)
localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */ localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */

35
src/libstore/machines.cc
View File

@ -6,7 +6,8 @@
namespace nix { namespace nix {
Machine::Machine(decltype(storeUri) storeUri, Machine::Machine(
const std::string & storeUri,
decltype(systemTypes) systemTypes, decltype(systemTypes) systemTypes,
decltype(sshKey) sshKey, decltype(sshKey) sshKey,
decltype(maxJobs) maxJobs, decltype(maxJobs) maxJobs,
@ -14,7 +15,7 @@ Machine::Machine(decltype(storeUri) storeUri,
decltype(supportedFeatures) supportedFeatures, decltype(supportedFeatures) supportedFeatures,
decltype(mandatoryFeatures) mandatoryFeatures, decltype(mandatoryFeatures) mandatoryFeatures,
decltype(sshPublicHostKey) sshPublicHostKey) : decltype(sshPublicHostKey) sshPublicHostKey) :
storeUri( storeUri(StoreReference::parse(
// Backwards compatibility: if the URI is schemeless, is not a path, // Backwards compatibility: if the URI is schemeless, is not a path,
// and is not one of the special store connection words, prepend // and is not one of the special store connection words, prepend
// ssh://. // ssh://.
@ -28,7 +29,7 @@ Machine::Machine(decltype(storeUri) storeUri,
|| hasPrefix(storeUri, "local?") || hasPrefix(storeUri, "local?")
|| hasPrefix(storeUri, "?") || hasPrefix(storeUri, "?")
? storeUri ? storeUri
: "ssh://" + storeUri), : "ssh://" + storeUri)),
systemTypes(systemTypes), systemTypes(systemTypes),
sshKey(sshKey), sshKey(sshKey),
maxJobs(maxJobs), maxJobs(maxJobs),
@ -63,23 +64,26 @@ bool Machine::mandatoryMet(const std::set<std::string> & features) const
}); });
} }
ref<Store> Machine::openStore() const StoreReference Machine::completeStoreReference() const
{ {
Store::Params storeParams; auto storeUri = this->storeUri;
if (hasPrefix(storeUri, "ssh://")) {
storeParams["max-connections"] = "1"; auto * generic = std::get_if<StoreReference::Specified>(&storeUri.variant);
storeParams["log-fd"] = "4";
if (generic && generic->scheme == "ssh") {
storeUri.params["max-connections"] = "1";
storeUri.params["log-fd"] = "4";
} }
if (hasPrefix(storeUri, "ssh://") || hasPrefix(storeUri, "ssh-ng://")) { if (generic && (generic->scheme == "ssh" || generic->scheme == "ssh-ng")) {
if (sshKey != "") if (sshKey != "")
storeParams["ssh-key"] = sshKey; storeUri.params["ssh-key"] = sshKey;
if (sshPublicHostKey != "") if (sshPublicHostKey != "")
storeParams["base64-ssh-public-host-key"] = sshPublicHostKey; storeUri.params["base64-ssh-public-host-key"] = sshPublicHostKey;
} }
{ {
auto & fs = storeParams["system-features"]; auto & fs = storeUri.params["system-features"];
auto append = [&](auto feats) { auto append = [&](auto feats) {
for (auto & f : feats) { for (auto & f : feats) {
if (fs.size() > 0) fs += ' '; if (fs.size() > 0) fs += ' ';
@ -90,7 +94,12 @@ ref<Store> Machine::openStore() const
append(mandatoryFeatures); append(mandatoryFeatures);
} }
return nix::openStore(storeUri, storeParams); return storeUri;
}
ref<Store> Machine::openStore() const
{
return nix::openStore(completeStoreReference());
} }
static std::vector<std::string> expandBuilderLines(const std::string & builders) static std::vector<std::string> expandBuilderLines(const std::string & builders)

21
src/libstore/machines.hh
View File

@ -2,6 +2,7 @@
///@file ///@file
#include "types.hh" #include "types.hh"
#include "store-reference.hh"
namespace nix { namespace nix {
@ -9,7 +10,7 @@ class Store;
struct Machine { struct Machine {
const std::string storeUri; const StoreReference storeUri;
const std::set<std::string> systemTypes; const std::set<std::string> systemTypes;
const std::string sshKey; const std::string sshKey;
const unsigned int maxJobs; const unsigned int maxJobs;
@ -36,7 +37,8 @@ struct Machine {
*/ */
bool mandatoryMet(const std::set<std::string> & features) const; bool mandatoryMet(const std::set<std::string> & features) const;
Machine(decltype(storeUri) storeUri, Machine(
const std::string & storeUri,
decltype(systemTypes) systemTypes, decltype(systemTypes) systemTypes,
decltype(sshKey) sshKey, decltype(sshKey) sshKey,
decltype(maxJobs) maxJobs, decltype(maxJobs) maxJobs,
@ -45,6 +47,21 @@ struct Machine {
decltype(mandatoryFeatures) mandatoryFeatures, decltype(mandatoryFeatures) mandatoryFeatures,
decltype(sshPublicHostKey) sshPublicHostKey); decltype(sshPublicHostKey) sshPublicHostKey);
/**
* Elaborate `storeUri` into a complete store reference,
* incorporating information from the other fields of the `Machine`
* as applicable.
*/
StoreReference completeStoreReference() const;
/**
* Open a `Store` for this machine.
*
* Just a simple function composition:
* ```c++
* nix::openStore(completeStoreReference())
* ```
*/
ref<Store> openStore() const; ref<Store> openStore() const;
}; };

47
tests/unit/libstore/machines.cc
View File

@ -3,24 +3,16 @@
#include "file-system.hh" #include "file-system.hh"
#include "util.hh" #include "util.hh"
#include <gtest/gtest.h>
#include <gmock/gmock-matchers.h> #include <gmock/gmock-matchers.h>
using testing::Contains; using testing::Contains;
using testing::ElementsAre; using testing::ElementsAre;
using testing::EndsWith;
using testing::Eq; using testing::Eq;
using testing::Field; using testing::Field;
using testing::SizeIs; using testing::SizeIs;
using nix::absPath; using namespace nix;
using nix::FormatError;
using nix::UsageError;
using nix::getMachines;
using nix::Machine;
using nix::Machines;
using nix::pathExists;
using nix::Settings;
using nix::settings;
class Environment : public ::testing::Environment { class Environment : public ::testing::Environment {
public: public:
@ -40,7 +32,7 @@ TEST(machines, getMachinesUriOnly) {
settings.builders = "nix@scratchy.labs.cs.uu.nl"; settings.builders = "nix@scratchy.labs.cs.uu.nl";
Machines actual = getMachines(); Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(1)); ASSERT_THAT(actual, SizeIs(1));
EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl"))); EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl"))));
EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS")));
EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0))); EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0)));
EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1)));
@ -54,7 +46,7 @@ TEST(machines, getMachinesDefaults) {
settings.builders = "nix@scratchy.labs.cs.uu.nl - - - - - - -"; settings.builders = "nix@scratchy.labs.cs.uu.nl - - - - - - -";
Machines actual = getMachines(); Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(1)); ASSERT_THAT(actual, SizeIs(1));
EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl"))); EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl"))));
EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS")));
EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0))); EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0)));
EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1)));
@ -64,20 +56,31 @@ TEST(machines, getMachinesDefaults) {
EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, SizeIs(0))); EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, SizeIs(0)));
} }
MATCHER_P(AuthorityMatches, authority, "") {
*result_listener
<< "where the authority of "
<< arg.render()
<< " is "
<< authority;
auto * generic = std::get_if<StoreReference::Specified>(&arg.variant);
if (!generic) return false;
return generic->authority == authority;
}
TEST(machines, getMachinesWithNewLineSeparator) { TEST(machines, getMachinesWithNewLineSeparator) {
settings.builders = "nix@scratchy.labs.cs.uu.nl\nnix@itchy.labs.cs.uu.nl"; settings.builders = "nix@scratchy.labs.cs.uu.nl\nnix@itchy.labs.cs.uu.nl";
Machines actual = getMachines(); Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(2)); ASSERT_THAT(actual, SizeIs(2));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl"))));
} }
TEST(machines, getMachinesWithSemicolonSeparator) { TEST(machines, getMachinesWithSemicolonSeparator) {
settings.builders = "nix@scratchy.labs.cs.uu.nl ; nix@itchy.labs.cs.uu.nl"; settings.builders = "nix@scratchy.labs.cs.uu.nl ; nix@itchy.labs.cs.uu.nl";
Machines actual = getMachines(); Machines actual = getMachines();
EXPECT_THAT(actual, SizeIs(2)); EXPECT_THAT(actual, SizeIs(2));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl"))));
} }
TEST(machines, getMachinesWithCorrectCompleteSingleBuilder) { TEST(machines, getMachinesWithCorrectCompleteSingleBuilder) {
@ -86,7 +89,7 @@ TEST(machines, getMachinesWithCorrectCompleteSingleBuilder) {
"benchmark SSH+HOST+PUBLIC+KEY+BASE64+ENCODED=="; "benchmark SSH+HOST+PUBLIC+KEY+BASE64+ENCODED==";
Machines actual = getMachines(); Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(1)); ASSERT_THAT(actual, SizeIs(1));
EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); EXPECT_THAT(actual[0], Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")));
EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux"))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux")));
EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto"))); EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto")));
EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8)));
@ -104,7 +107,7 @@ TEST(machines,
"KEY+BASE64+ENCODED=="; "KEY+BASE64+ENCODED==";
Machines actual = getMachines(); Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(1)); ASSERT_THAT(actual, SizeIs(1));
EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); EXPECT_THAT(actual[0], Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")));
EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux"))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux")));
EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto"))); EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto")));
EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8)));
@ -120,7 +123,7 @@ TEST(machines, getMachinesWithMultiOptions) {
"MandatoryFeature1,MandatoryFeature2"; "MandatoryFeature1,MandatoryFeature2";
Machines actual = getMachines(); Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(1)); ASSERT_THAT(actual, SizeIs(1));
EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); EXPECT_THAT(actual[0], Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")));
EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("Arch1", "Arch2"))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("Arch1", "Arch2")));
EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("SupportedFeature1", "SupportedFeature2"))); EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("SupportedFeature1", "SupportedFeature2")));
EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("MandatoryFeature1", "MandatoryFeature2"))); EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("MandatoryFeature1", "MandatoryFeature2")));
@ -146,9 +149,9 @@ TEST(machines, getMachinesWithCorrectFileReference) {
settings.builders = std::string("@") + path; settings.builders = std::string("@") + path;
Machines actual = getMachines(); Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(3)); ASSERT_THAT(actual, SizeIs(3));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl"))));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@poochie.labs.cs.uu.nl")))); EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@poochie.labs.cs.uu.nl"))));
} }
TEST(machines, getMachinesWithCorrectFileReferenceToEmptyFile) { TEST(machines, getMachinesWithCorrectFileReferenceToEmptyFile) {