From 39b2a399ad94f061eea0e0fc639cf1466587959e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 10 May 2024 13:03:05 -0400 Subject: [PATCH] Start building the scheduler for Windows Building derivations is a lot harder, but the downloading goals is portable enough. The "common channel" code is due to Volth. I wonder if there is a way we can factor it out into separate functions / files to avoid some within-function CPP. Co-authored-by: volth --- maintainers/flake-module.nix | 22 ++-- .../build/drv-output-substitution-goal.cc | 14 ++- .../build/drv-output-substitution-goal.hh | 15 ++- src/libstore/{unix => }/build/entry-points.cc | 14 ++- src/libstore/{unix => }/build/goal.cc | 0 src/libstore/{unix => }/build/goal.hh | 4 +- .../{unix => }/build/substitution-goal.cc | 16 ++- .../{unix => }/build/substitution-goal.hh | 13 ++- src/libstore/{unix => }/build/worker.cc | 110 ++++++++++++++++-- src/libstore/{unix => }/build/worker.hh | 33 +++++- src/libstore/local.mk | 4 +- src/libstore/unix/build/derivation-goal.cc | 4 +- src/libstore/unix/build/derivation-goal.hh | 6 +- src/libstore/unix/{lock.cc => user-lock.cc} | 2 +- src/libstore/unix/{lock.hh => user-lock.hh} | 0 src/libstore/windows/build.cc | 37 ------ src/libstore/windows/pathlocks.cc | 2 + src/libutil/error.hh | 10 +- src/libutil/serialise.cc | 2 +- src/libutil/windows/file-descriptor.cc | 2 + src/libutil/windows/users.cc | 2 + src/libutil/windows/windows-async-pipe.cc | 43 +++++++ src/libutil/windows/windows-async-pipe.hh | 20 ++++ src/libutil/windows/windows-error.cc | 2 +- src/libutil/windows/windows-error.hh | 2 +- 25 files changed, 285 insertions(+), 94 deletions(-) rename src/libstore/{unix => }/build/drv-output-substitution-goal.cc (93%) rename src/libstore/{unix => }/build/drv-output-substitution-goal.hh (90%) rename src/libstore/{unix => }/build/entry-points.cc (90%) rename src/libstore/{unix => }/build/goal.cc (100%) rename src/libstore/{unix => }/build/goal.hh (97%) rename src/libstore/{unix => }/build/substitution-goal.cc (96%) rename src/libstore/{unix => }/build/substitution-goal.hh (91%) rename src/libstore/{unix => }/build/worker.cc (81%) rename src/libstore/{unix => }/build/worker.hh (92%) rename src/libstore/unix/{lock.cc => user-lock.cc} (99%) rename src/libstore/unix/{lock.hh => user-lock.hh} (100%) delete mode 100644 src/libstore/windows/build.cc create mode 100644 src/libutil/windows/windows-async-pipe.cc create mode 100644 src/libutil/windows/windows-async-pipe.hh diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 351a01fcb..e541aeda8 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -227,18 +227,18 @@ ''^src/libstore/store-dir-config\.hh$'' ''^src/libstore/unix/build/derivation-goal\.cc$'' ''^src/libstore/unix/build/derivation-goal\.hh$'' - ''^src/libstore/unix/build/drv-output-substitution-goal\.cc$'' - ''^src/libstore/unix/build/drv-output-substitution-goal\.hh$'' - ''^src/libstore/unix/build/entry-points\.cc$'' - ''^src/libstore/unix/build/goal\.cc$'' - ''^src/libstore/unix/build/goal\.hh$'' + ''^src/libstore/build/drv-output-substitution-goal\.cc$'' + ''^src/libstore/build/drv-output-substitution-goal\.hh$'' + ''^src/libstore/build/entry-points\.cc$'' + ''^src/libstore/build/goal\.cc$'' + ''^src/libstore/build/goal\.hh$'' ''^src/libstore/unix/build/hook-instance\.cc$'' ''^src/libstore/unix/build/local-derivation-goal\.cc$'' ''^src/libstore/unix/build/local-derivation-goal\.hh$'' - ''^src/libstore/unix/build/substitution-goal\.cc$'' - ''^src/libstore/unix/build/substitution-goal\.hh$'' - ''^src/libstore/unix/build/worker\.cc$'' - ''^src/libstore/unix/build/worker\.hh$'' + ''^src/libstore/build/substitution-goal\.cc$'' + ''^src/libstore/build/substitution-goal\.hh$'' + ''^src/libstore/build/worker\.cc$'' + ''^src/libstore/build/worker\.hh$'' ''^src/libstore/unix/builtins/fetchurl\.cc$'' ''^src/libstore/unix/builtins/unpack-channel\.cc$'' ''^src/libstore/gc\.cc$'' @@ -246,8 +246,8 @@ ''^src/libstore/unix/local-overlay-store\.hh$'' ''^src/libstore/local-store\.cc$'' ''^src/libstore/local-store\.hh$'' - ''^src/libstore/unix/lock\.cc$'' - ''^src/libstore/unix/lock\.hh$'' + ''^src/libstore/unix/user-lock\.cc$'' + ''^src/libstore/unix/user-lock\.hh$'' ''^src/libstore/optimise-store\.cc$'' ''^src/libstore/unix/pathlocks\.cc$'' ''^src/libstore/posix-fs-canonicalise\.cc$'' diff --git a/src/libstore/unix/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc similarity index 93% rename from src/libstore/unix/build/drv-output-substitution-goal.cc rename to src/libstore/build/drv-output-substitution-goal.cc index b30957c84..13a07e4ea 100644 --- a/src/libstore/unix/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -66,7 +66,11 @@ void DrvOutputSubstitutionGoal::tryNext() some other error occurs), so it must not touch `this`. So put the shared state in a separate refcounted object. */ downloadState = std::make_shared(); +#ifndef _WIN32 downloadState->outPipe.create(); +#else + downloadState->outPipe.createAsyncPipe(worker.ioport.get()); +#endif sub->queryRealisation( id, @@ -79,7 +83,13 @@ void DrvOutputSubstitutionGoal::tryNext() } } }); - worker.childStarted(shared_from_this(), {downloadState->outPipe.readSide.get()}, true, false); + worker.childStarted(shared_from_this(), { +#ifndef _WIN32 + downloadState->outPipe.readSide.get() +#else + &downloadState->outPipe +#endif + }, true, false); state = &DrvOutputSubstitutionGoal::realisationFetched; } @@ -158,7 +168,7 @@ void DrvOutputSubstitutionGoal::work() (this->*state)(); } -void DrvOutputSubstitutionGoal::handleEOF(int fd) +void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd) { if (fd == downloadState->outPipe.readSide.get()) worker.wakeUp(shared_from_this()); } diff --git a/src/libstore/unix/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh similarity index 90% rename from src/libstore/unix/build/drv-output-substitution-goal.hh rename to src/libstore/build/drv-output-substitution-goal.hh index da2426e5e..4f06a9e9e 100644 --- a/src/libstore/unix/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -1,11 +1,16 @@ #pragma once ///@file +#include +#include + #include "store-api.hh" #include "goal.hh" #include "realisation.hh" -#include -#include + +#ifdef _WIN32 +# include "windows-async-pipe.hh" +#endif namespace nix { @@ -43,7 +48,11 @@ class DrvOutputSubstitutionGoal : public Goal { struct DownloadState { +#ifndef _WIN32 Pipe outPipe; +#else + windows::AsyncPipe outPipe; +#endif std::promise> promise; }; @@ -71,7 +80,7 @@ public: std::string key() override; void work() override; - void handleEOF(int fd) override; + void handleEOF(Descriptor fd) override; JobCategory jobCategory() const override { return JobCategory::Substitution; diff --git a/src/libstore/unix/build/entry-points.cc b/src/libstore/build/entry-points.cc similarity index 90% rename from src/libstore/unix/build/entry-points.cc rename to src/libstore/build/entry-points.cc index d4bead28e..784f618c1 100644 --- a/src/libstore/unix/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -1,6 +1,8 @@ #include "worker.hh" #include "substitution-goal.hh" -#include "derivation-goal.hh" +#ifndef _WIN32 // TODO Enable building on Windows +# include "derivation-goal.hh" +#endif #include "local-store.hh" namespace nix { @@ -25,9 +27,12 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod ex = std::move(i->ex); } if (i->exitCode != Goal::ecSuccess) { +#ifndef _WIN32 // TODO Enable building on Windows if (auto i2 = dynamic_cast(i.get())) failed.insert(printStorePath(i2->drvPath)); - else if (auto i2 = dynamic_cast(i.get())) + else +#endif + if (auto i2 = dynamic_cast(i.get())) failed.insert(printStorePath(i2->storePath)); } } @@ -74,7 +79,12 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat BuildMode buildMode) { Worker worker(*this, *this); +#ifndef _WIN32 // TODO Enable building on Windows auto goal = worker.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All {}, buildMode); +#else + std::shared_ptr goal; + throw UnimplementedError("Building derivations not yet implemented on windows."); +#endif try { worker.run(Goals{goal}); diff --git a/src/libstore/unix/build/goal.cc b/src/libstore/build/goal.cc similarity index 100% rename from src/libstore/unix/build/goal.cc rename to src/libstore/build/goal.cc diff --git a/src/libstore/unix/build/goal.hh b/src/libstore/build/goal.hh similarity index 97% rename from src/libstore/unix/build/goal.hh rename to src/libstore/build/goal.hh index 9af083230..0d9b828e1 100644 --- a/src/libstore/unix/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -138,12 +138,12 @@ public: virtual void waiteeDone(GoalPtr waitee, ExitCode result); - virtual void handleChildOutput(int fd, std::string_view data) + virtual void handleChildOutput(Descriptor fd, std::string_view data) { abort(); } - virtual void handleEOF(int fd) + virtual void handleEOF(Descriptor fd) { abort(); } diff --git a/src/libstore/unix/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc similarity index 96% rename from src/libstore/unix/build/substitution-goal.cc rename to src/libstore/build/substitution-goal.cc index c7e8e2825..0be3d1e8d 100644 --- a/src/libstore/unix/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -212,7 +212,11 @@ void PathSubstitutionGoal::tryToRun() maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); worker.updateProgress(); +#ifndef _WIN32 outPipe.create(); +#else + outPipe.createAsyncPipe(worker.ioport.get()); +#endif promise = std::promise(); @@ -235,7 +239,13 @@ void PathSubstitutionGoal::tryToRun() } }); - worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); + worker.childStarted(shared_from_this(), { +#ifndef _WIN32 + outPipe.readSide.get() +#else + &outPipe +#endif + }, true, false); state = &PathSubstitutionGoal::finished; } @@ -294,12 +304,12 @@ void PathSubstitutionGoal::finished() } -void PathSubstitutionGoal::handleChildOutput(int fd, std::string_view data) +void PathSubstitutionGoal::handleChildOutput(Descriptor fd, std::string_view data) { } -void PathSubstitutionGoal::handleEOF(int fd) +void PathSubstitutionGoal::handleEOF(Descriptor fd) { if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); } diff --git a/src/libstore/unix/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh similarity index 91% rename from src/libstore/unix/build/substitution-goal.hh rename to src/libstore/build/substitution-goal.hh index 1d389d328..58d309424 100644 --- a/src/libstore/unix/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -1,10 +1,13 @@ #pragma once ///@file -#include "lock.hh" #include "store-api.hh" #include "goal.hh" +#ifdef _WIN32 +# include "windows-async-pipe.hh" +#endif + namespace nix { class Worker; @@ -45,7 +48,11 @@ struct PathSubstitutionGoal : public Goal /** * Pipe for the substituter's standard output. */ +#ifndef _WIN32 Pipe outPipe; +#else + windows::AsyncPipe outPipe; +#endif /** * The substituter thread. @@ -111,8 +118,8 @@ public: /** * Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, std::string_view data) override; - void handleEOF(int fd) override; + void handleChildOutput(Descriptor fd, std::string_view data) override; + void handleEOF(Descriptor fd) override; /* Called by destructor, can't be overridden */ void cleanup() override final; diff --git a/src/libstore/unix/build/worker.cc b/src/libstore/build/worker.cc similarity index 81% rename from src/libstore/unix/build/worker.cc rename to src/libstore/build/worker.cc index 03fc280a4..ac6f693a2 100644 --- a/src/libstore/unix/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -1,12 +1,20 @@ +#include "local-store.hh" #include "machines.hh" #include "worker.hh" #include "substitution-goal.hh" #include "drv-output-substitution-goal.hh" -#include "local-derivation-goal.hh" -#include "hook-instance.hh" +#ifndef _WIN32 // TODO Enable building on Windows +# include "local-derivation-goal.hh" +# include "hook-instance.hh" +#endif #include "signals.hh" -#include +#ifndef _WIN32 +# include +#else +# include +# include "windows-error.hh" +#endif namespace nix { @@ -41,6 +49,7 @@ Worker::~Worker() assert(expectedNarSize == 0); } +#ifndef _WIN32 // TODO Enable building on Windows std::shared_ptr Worker::makeDerivationGoalCommon( const StorePath & drvPath, @@ -70,7 +79,6 @@ std::shared_ptr Worker::makeDerivationGoal(const StorePath & drv }); } - std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode) { @@ -81,6 +89,8 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath }); } +#endif + std::shared_ptr Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) { @@ -112,10 +122,14 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) { return std::visit(overloaded { [&](const DerivedPath::Built & bfd) -> GoalPtr { +#ifndef _WIN32 // TODO Enable building on Windows if (auto bop = std::get_if(&*bfd.drvPath)) return makeDerivationGoal(bop->path, bfd.outputs, buildMode); else throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented."); +#else + throw UnimplementedError("Building derivations not yet implemented on Windows"); +#endif }, [&](const DerivedPath::Opaque & bo) -> GoalPtr { return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); @@ -141,9 +155,12 @@ static void removeGoal(std::shared_ptr goal, std::map> & void Worker::removeGoal(GoalPtr goal) { +#ifndef _WIN32 // TODO Enable building on Windows if (auto drvGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(drvGoal, derivationGoals); - else if (auto subGoal = std::dynamic_pointer_cast(goal)) + else +#endif + if (auto subGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(subGoal, substitutionGoals); else if (auto subGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(subGoal, drvOutputSubstitutionGoals); @@ -187,13 +204,13 @@ unsigned Worker::getNrSubstitutions() } -void Worker::childStarted(GoalPtr goal, const std::set & fds, +void Worker::childStarted(GoalPtr goal, const std::set & channels, bool inBuildSlot, bool respectTimeouts) { Child child; child.goal = goal; child.goal2 = goal.get(); - child.fds = fds; + child.channels = channels; child.timeStarted = child.lastOutput = steady_time_point::clock::now(); child.inBuildSlot = inBuildSlot; child.respectTimeouts = respectTimeouts; @@ -281,12 +298,15 @@ void Worker::run(const Goals & _topGoals) for (auto & i : _topGoals) { topGoals.insert(i); +#ifndef _WIN32 // TODO Enable building on Windows if (auto goal = dynamic_cast(i.get())) { topPaths.push_back(DerivedPath::Built { .drvPath = makeConstantStorePathRef(goal->drvPath), .outputs = goal->wantedOutputs, }); - } else if (auto goal = dynamic_cast(i.get())) { + } else +#endif + if (auto goal = dynamic_cast(i.get())) { topPaths.push_back(DerivedPath::Opaque{goal->storePath}); } } @@ -408,13 +428,14 @@ void Worker::waitForInput() if (useTimeout) vomit("sleeping %d seconds", timeout); +#ifndef _WIN32 /* Use select() to wait for the input side of any logger pipe to become `available'. Note that `available' (i.e., non-blocking) includes EOF. */ std::vector pollStatus; std::map fdToPollStatus; for (auto & i : children) { - for (auto & j : i.fds) { + for (auto & j : i.channels) { pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); fdToPollStatus[j] = pollStatus.size() - 1; } @@ -425,6 +446,28 @@ void Worker::waitForInput() if (errno == EINTR) return; throw SysError("waiting for input"); } +#else + OVERLAPPED_ENTRY oentries[0x20] = {0}; + ULONG removed; + bool gotEOF = false; + + // we are on at least Windows Vista / Server 2008 and can get many (countof(oentries)) statuses in one API call + if (!GetQueuedCompletionStatusEx( + ioport.get(), + oentries, + sizeof(oentries) / sizeof(*oentries), + &removed, + useTimeout ? timeout * 1000 : INFINITE, + false)) + { + windows::WinError winError("GetQueuedCompletionStatusEx"); + if (winError.lastError != WAIT_TIMEOUT) + throw winError; + assert(removed == 0); + } else { + assert(0 < removed && removed <= sizeof(oentries)/sizeof(*oentries)); + } +#endif auto after = steady_time_point::clock::now(); @@ -439,20 +482,21 @@ void Worker::waitForInput() GoalPtr goal = j->goal.lock(); assert(goal); - std::set fds2(j->fds); +#ifndef _WIN32 + std::set fds2(j->channels); std::vector buffer(4096); for (auto & k : fds2) { const auto fdPollStatusId = get(fdToPollStatus, k); assert(fdPollStatusId); assert(*fdPollStatusId < pollStatus.size()); if (pollStatus.at(*fdPollStatusId).revents) { - ssize_t rd = ::read(k, buffer.data(), buffer.size()); + ssize_t rd = ::read(fromDescriptorReadOnly(k), buffer.data(), buffer.size()); // FIXME: is there a cleaner way to handle pt close // than EIO? Is this even standard? if (rd == 0 || (rd == -1 && errno == EIO)) { debug("%1%: got EOF", goal->getName()); goal->handleEOF(k); - j->fds.erase(k); + j->channels.erase(k); } else if (rd == -1) { if (errno != EINTR) throw SysError("%s: read failed", goal->getName()); @@ -465,6 +509,48 @@ void Worker::waitForInput() } } } +#else + decltype(j->channels)::iterator p = j->channels.begin(); + while (p != j->channels.end()) { + decltype(p) nextp = p; + ++nextp; + for (ULONG i = 0; i < removed; i++) { + if (oentries[i].lpCompletionKey == ((ULONG_PTR)((*p)->readSide.get()) ^ 0x5555)) { + printMsg(lvlVomit, "%s: read %s bytes", goal->getName(), oentries[i].dwNumberOfBytesTransferred); + if (oentries[i].dwNumberOfBytesTransferred > 0) { + std::string data { + (char *) (*p)->buffer.data(), + oentries[i].dwNumberOfBytesTransferred, + }; + //std::cerr << "read [" << data << "]" << std::endl; + j->lastOutput = after; + goal->handleChildOutput((*p)->readSide.get(), data); + } + + if (gotEOF) { + debug("%s: got EOF", goal->getName()); + goal->handleEOF((*p)->readSide.get()); + nextp = j->channels.erase(p); // no need to maintain `j->channels` ? + } else { + BOOL rc = ReadFile((*p)->readSide.get(), (*p)->buffer.data(), (*p)->buffer.size(), &(*p)->got, &(*p)->overlapped); + if (rc) { + // here is possible (but not obligatory) to call `goal->handleChildOutput` and repeat ReadFile immediately + } else { + windows::WinError winError("ReadFile(%s, ..)", (*p)->readSide.get()); + if (winError.lastError == ERROR_BROKEN_PIPE) { + debug("%s: got EOF", goal->getName()); + goal->handleEOF((*p)->readSide.get()); + nextp = j->channels.erase(p); // no need to maintain `j->channels` ? + } else if (winError.lastError != ERROR_IO_PENDING) + throw winError; + } + } + break; + } + } + p = nextp; + } +#endif if (goal->exitCode == Goal::ecBusy && 0 != settings.maxSilentTime && diff --git a/src/libstore/unix/build/worker.hh b/src/libstore/build/worker.hh similarity index 92% rename from src/libstore/unix/build/worker.hh rename to src/libstore/build/worker.hh index ced013ddd..b57734b45 100644 --- a/src/libstore/unix/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -2,18 +2,23 @@ ///@file #include "types.hh" -#include "lock.hh" #include "store-api.hh" #include "goal.hh" #include "realisation.hh" +#ifdef _WIN32 +# include "windows-async-pipe.hh" +#endif + #include #include namespace nix { /* Forward definition. */ +#ifndef _WIN32 // TODO Enable building on Windows struct DerivationGoal; +#endif struct PathSubstitutionGoal; class DrvOutputSubstitutionGoal; @@ -36,14 +41,22 @@ typedef std::chrono::time_point steady_time_point; /** * A mapping used to remember for each child process to what goal it - * belongs, and file descriptors for receiving log data and output + * belongs, and comm channels for receiving log data and output * path creation commands. */ struct Child { + using CommChannel = +#ifndef _WIN32 + Descriptor +#else + windows::AsyncPipe * +#endif + ; + WeakGoalPtr goal; Goal * goal2; // ugly hackery - std::set fds; + std::set channels; bool respectTimeouts; bool inBuildSlot; /** @@ -53,8 +66,10 @@ struct Child steady_time_point timeStarted; }; +#ifndef _WIN32 // TODO Enable building on Windows /* Forward definition. */ struct HookInstance; +#endif /** * The worker class. @@ -101,7 +116,9 @@ private: * Maps used to prevent multiple instantiations of a goal for the * same derivation / path. */ +#ifndef _WIN32 // TODO Enable building on Windows std::map> derivationGoals; +#endif std::map> substitutionGoals; std::map> drvOutputSubstitutionGoals; @@ -152,10 +169,16 @@ public: */ bool checkMismatch; +#ifdef _WIN32 + AutoCloseFD ioport; +#endif + Store & store; Store & evalStore; +#ifndef _WIN32 // TODO Enable building on Windows std::unique_ptr hook; +#endif uint64_t expectedBuilds = 0; uint64_t doneBuilds = 0; @@ -184,6 +207,7 @@ public: * Make a goal (with caching). */ +#ifndef _WIN32 // TODO Enable building on Windows /** * @ref DerivationGoal "derivation goal" */ @@ -198,6 +222,7 @@ public: std::shared_ptr makeBasicDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); +#endif /** * @ref SubstitutionGoal "substitution goal" @@ -238,7 +263,7 @@ public: * Registers a running child process. `inBuildSlot` means that * the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, const std::set & fds, + void childStarted(GoalPtr goal, const std::set & channels, bool inBuildSlot, bool respectTimeouts); /** diff --git a/src/libstore/local.mk b/src/libstore/local.mk index cc67da786..60174d009 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -4,7 +4,7 @@ libstore_NAME = libnixstore libstore_DIR := $(d) -libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) +libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc) ifdef HOST_UNIX libstore_SOURCES += $(wildcard $(d)/unix/*.cc $(d)/unix/builtins/*.cc $(d)/unix/build/*.cc) endif @@ -43,7 +43,7 @@ endif INCLUDE_libstore := -I $(d) -I $(d)/build ifdef HOST_UNIX - INCLUDE_libstore += -I $(d)/unix + INCLUDE_libstore += -I $(d)/unix -I $(d)/unix/build endif ifdef HOST_LINUX INCLUDE_libstore += -I $(d)/linux diff --git a/src/libstore/unix/build/derivation-goal.cc b/src/libstore/unix/build/derivation-goal.cc index 8d6e35015..004094015 100644 --- a/src/libstore/unix/build/derivation-goal.cc +++ b/src/libstore/unix/build/derivation-goal.cc @@ -1280,7 +1280,7 @@ bool DerivationGoal::isReadDesc(int fd) return fd == hook->builderOut.readSide.get(); } -void DerivationGoal::handleChildOutput(int fd, std::string_view data) +void DerivationGoal::handleChildOutput(Descriptor fd, std::string_view data) { // local & `ssh://`-builds are dealt with here. auto isWrittenToLog = isReadDesc(fd); @@ -1347,7 +1347,7 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data) } -void DerivationGoal::handleEOF(int fd) +void DerivationGoal::handleEOF(Descriptor fd) { if (!currentLogLine.empty()) flushLine(); worker.wakeUp(shared_from_this()); diff --git a/src/libstore/unix/build/derivation-goal.hh b/src/libstore/unix/build/derivation-goal.hh index ddb5ee1e3..be6eb50c4 100644 --- a/src/libstore/unix/build/derivation-goal.hh +++ b/src/libstore/unix/build/derivation-goal.hh @@ -2,7 +2,7 @@ ///@file #include "parsed-derivations.hh" -#include "lock.hh" +#include "user-lock.hh" #include "outputs-spec.hh" #include "store-api.hh" #include "pathlocks.hh" @@ -292,8 +292,8 @@ struct DerivationGoal : public Goal /** * Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, std::string_view data) override; - void handleEOF(int fd) override; + void handleChildOutput(Descriptor fd, std::string_view data) override; + void handleEOF(Descriptor fd) override; void flushLine(); /** diff --git a/src/libstore/unix/lock.cc b/src/libstore/unix/user-lock.cc similarity index 99% rename from src/libstore/unix/lock.cc rename to src/libstore/unix/user-lock.cc index 023c74e34..8057aa13e 100644 --- a/src/libstore/unix/lock.cc +++ b/src/libstore/unix/user-lock.cc @@ -1,4 +1,4 @@ -#include "lock.hh" +#include "user-lock.hh" #include "file-system.hh" #include "globals.hh" #include "pathlocks.hh" diff --git a/src/libstore/unix/lock.hh b/src/libstore/unix/user-lock.hh similarity index 100% rename from src/libstore/unix/lock.hh rename to src/libstore/unix/user-lock.hh diff --git a/src/libstore/windows/build.cc b/src/libstore/windows/build.cc deleted file mode 100644 index 3eadc5bda..000000000 --- a/src/libstore/windows/build.cc +++ /dev/null @@ -1,37 +0,0 @@ -#include "store-api.hh" -#include "build-result.hh" - -namespace nix { - -void Store::buildPaths(const std::vector & reqs, BuildMode buildMode, std::shared_ptr evalStore) -{ - unsupported("buildPaths"); -} - -std::vector Store::buildPathsWithResults( - const std::vector & reqs, - BuildMode buildMode, - std::shared_ptr evalStore) -{ - unsupported("buildPathsWithResults"); -} - -BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) -{ - unsupported("buildDerivation"); -} - - -void Store::ensurePath(const StorePath & path) -{ - unsupported("ensurePath"); -} - - -void Store::repairPath(const StorePath & path) -{ - unsupported("repairPath"); -} - -} diff --git a/src/libstore/windows/pathlocks.cc b/src/libstore/windows/pathlocks.cc index 738057f68..8a4d55e00 100644 --- a/src/libstore/windows/pathlocks.cc +++ b/src/libstore/windows/pathlocks.cc @@ -9,6 +9,8 @@ namespace nix { +using namespace nix::windows; + void deleteLockFile(const Path & path, Descriptor desc) { diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 0419f36d6..87d181c94 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -206,11 +206,11 @@ MakeError(SystemError, Error); * * Throw this, but prefer not to catch this, and catch `SystemError` * instead. This allows implementations to freely switch between this - * and `WinError` without breaking catch blocks. + * and `windows::WinError` without breaking catch blocks. * * However, it is permissible to catch this and rethrow so long as * certain conditions are not met (e.g. to catch only if `errNo = - * EFooBar`). In that case, try to also catch the equivalent `WinError` + * EFooBar`). In that case, try to also catch the equivalent `windows::WinError` * code. * * @todo Rename this to `PosixError` or similar. At this point Windows @@ -248,7 +248,9 @@ public: }; #ifdef _WIN32 -class WinError; +namespace windows { + class WinError; +} #endif /** @@ -258,7 +260,7 @@ class WinError; */ using NativeSysError = #ifdef _WIN32 - WinError + windows::WinError #else SysError #endif diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 5ea27ccbe..36b99905a 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -136,7 +136,7 @@ size_t FdSource::readUnbuffered(char * data, size_t len) checkInterrupt(); if (!::ReadFile(fd, data, len, &n, NULL)) { _good = false; - throw WinError("ReadFile when FdSource::readUnbuffered"); + throw windows::WinError("ReadFile when FdSource::readUnbuffered"); } #else ssize_t n; diff --git a/src/libutil/windows/file-descriptor.cc b/src/libutil/windows/file-descriptor.cc index 26f769b66..b5c21ad32 100644 --- a/src/libutil/windows/file-descriptor.cc +++ b/src/libutil/windows/file-descriptor.cc @@ -14,6 +14,8 @@ namespace nix { +using namespace nix::windows; + std::string readFile(HANDLE handle) { LARGE_INTEGER li; diff --git a/src/libutil/windows/users.cc b/src/libutil/windows/users.cc index 1792ff1a1..db6c42df3 100644 --- a/src/libutil/windows/users.cc +++ b/src/libutil/windows/users.cc @@ -9,6 +9,8 @@ namespace nix { +using namespace nix::windows; + std::string getUserName() { // Get the required buffer size diff --git a/src/libutil/windows/windows-async-pipe.cc b/src/libutil/windows/windows-async-pipe.cc new file mode 100644 index 000000000..7aae603bd --- /dev/null +++ b/src/libutil/windows/windows-async-pipe.cc @@ -0,0 +1,43 @@ +#include "windows-async-pipe.hh" +#include "windows-error.hh" + +namespace nix::windows { + +void AsyncPipe::createAsyncPipe(HANDLE iocp) +{ + // std::cerr << (format("-----AsyncPipe::createAsyncPipe(%x)") % iocp) << std::endl; + + buffer.resize(0x1000); + memset(&overlapped, 0, sizeof(overlapped)); + + std::string pipeName = fmt("\\\\.\\pipe\\nix-%d-%p", GetCurrentProcessId(), (void *) this); + + readSide = CreateNamedPipeA( + pipeName.c_str(), PIPE_ACCESS_INBOUND | FILE_FLAG_OVERLAPPED, PIPE_TYPE_BYTE, PIPE_UNLIMITED_INSTANCES, 0, 0, + INFINITE, NULL); + if (!readSide) + throw WinError("CreateNamedPipeA(%s)", pipeName); + + HANDLE hIocp = CreateIoCompletionPort(readSide.get(), iocp, (ULONG_PTR) (readSide.get()) ^ 0x5555, 0); + if (hIocp != iocp) + throw WinError("CreateIoCompletionPort(%x[%s], %x, ...) returned %x", readSide.get(), pipeName, iocp, hIocp); + + if (!ConnectNamedPipe(readSide.get(), &overlapped) && GetLastError() != ERROR_IO_PENDING) + throw WinError("ConnectNamedPipe(%s)", pipeName); + + SECURITY_ATTRIBUTES psa2 = {0}; + psa2.nLength = sizeof(SECURITY_ATTRIBUTES); + psa2.bInheritHandle = TRUE; + + writeSide = CreateFileA(pipeName.c_str(), GENERIC_WRITE, 0, &psa2, OPEN_EXISTING, 0, NULL); + if (!readSide) + throw WinError("CreateFileA(%s)", pipeName); +} + +void AsyncPipe::close() +{ + readSide.close(); + writeSide.close(); +} + +} diff --git a/src/libutil/windows/windows-async-pipe.hh b/src/libutil/windows/windows-async-pipe.hh new file mode 100644 index 000000000..c980201a8 --- /dev/null +++ b/src/libutil/windows/windows-async-pipe.hh @@ -0,0 +1,20 @@ +#pragma once +///@file + +#include "file-descriptor.hh" + +namespace nix::windows { + +class AsyncPipe +{ +public: + AutoCloseFD writeSide, readSide; + OVERLAPPED overlapped; + DWORD got; + std::vector buffer; + + void createAsyncPipe(HANDLE iocp); + void close(); +}; + +} diff --git a/src/libutil/windows/windows-error.cc b/src/libutil/windows/windows-error.cc index 26faaae6d..aead4af23 100644 --- a/src/libutil/windows/windows-error.cc +++ b/src/libutil/windows/windows-error.cc @@ -4,7 +4,7 @@ #define WIN32_LEAN_AND_MEAN #include -namespace nix { +namespace nix::windows { std::string WinError::renderError(DWORD lastError) { diff --git a/src/libutil/windows/windows-error.hh b/src/libutil/windows/windows-error.hh index fdfd0f52c..624b4c4cb 100644 --- a/src/libutil/windows/windows-error.hh +++ b/src/libutil/windows/windows-error.hh @@ -5,7 +5,7 @@ #include "error.hh" -namespace nix { +namespace nix::windows { /** * Windows Error type.