mirror of
https://github.com/NixOS/nix.git
synced 2024-11-22 06:42:28 +00:00
Merge pull request #10674 from nix-windows/local-store-on-windows
Build the local store on Windows
This commit is contained in:
John Ericson
GitHub
commit
87ab3c0ea4
@ -241,17 +241,17 @@
|
||||
''^src/libstore/unix/build/worker\.hh$''
|
||||
''^src/libstore/unix/builtins/fetchurl\.cc$''
|
||||
''^src/libstore/unix/builtins/unpack-channel\.cc$''
|
||||
''^src/libstore/unix/gc\.cc$''
|
||||
''^src/libstore/gc\.cc$''
|
||||
''^src/libstore/unix/local-overlay-store\.cc$''
|
||||
''^src/libstore/unix/local-overlay-store\.hh$''
|
||||
''^src/libstore/unix/local-store\.cc$''
|
||||
''^src/libstore/unix/local-store\.hh$''
|
||||
''^src/libstore/local-store\.cc$''
|
||||
''^src/libstore/local-store\.hh$''
|
||||
''^src/libstore/unix/lock\.cc$''
|
||||
''^src/libstore/unix/lock\.hh$''
|
||||
''^src/libstore/unix/optimise-store\.cc$''
|
||||
''^src/libstore/optimise-store\.cc$''
|
||||
''^src/libstore/unix/pathlocks\.cc$''
|
||||
''^src/libstore/unix/posix-fs-canonicalise\.cc$''
|
||||
''^src/libstore/unix/posix-fs-canonicalise\.hh$''
|
||||
''^src/libstore/posix-fs-canonicalise\.cc$''
|
||||
''^src/libstore/posix-fs-canonicalise\.hh$''
|
||||
''^src/libstore/uds-remote-store\.cc$''
|
||||
''^src/libstore/uds-remote-store\.hh$''
|
||||
''^src/libstore/windows/build\.cc$''
|
||||
|
||||
@ -109,7 +109,7 @@ struct CacheImpl : Cache
|
||||
Key key,
|
||||
Store & store,
|
||||
Attrs value,
|
||||
const StorePath & storePath)
|
||||
const StorePath & storePath) override
|
||||
{
|
||||
/* Add the store prefix to the cache key to handle multiple
|
||||
store prefixes. */
|
||||
|
||||
@ -19,18 +19,20 @@
|
||||
#include <climits>
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <poll.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/statvfs.h>
|
||||
#if HAVE_STATVFS
|
||||
# include <sys/statvfs.h>
|
||||
#endif
|
||||
#ifndef _WIN32
|
||||
# include <poll.h>
|
||||
# include <sys/socket.h>
|
||||
# include <sys/un.h>
|
||||
#endif
|
||||
#include <sys/types.h>
|
||||
#include <sys/un.h>
|
||||
#include <unistd.h>
|
||||
|
||||
namespace nix {
|
||||
|
||||
using namespace nix::unix;
|
||||
|
||||
static std::string gcSocketPath = "/gc-socket/socket";
|
||||
static std::string gcRootsDir = "gcroots";
|
||||
|
||||
@ -64,7 +66,7 @@ void LocalStore::createTempRootsFile()
|
||||
/* Check whether the garbage collector didn't get in our
|
||||
way. */
|
||||
struct stat st;
|
||||
if (fstat(fdTempRoots->get(), &st) == -1)
|
||||
if (fstat(fromDescriptorReadOnly(fdTempRoots->get()), &st) == -1)
|
||||
throw SysError("statting '%1%'", fnTempRoots);
|
||||
if (st.st_size == 0) break;
|
||||
|
||||
@ -108,7 +110,7 @@ void LocalStore::addTempRoot(const StorePath & path)
|
||||
debug("connecting to '%s'", socketPath);
|
||||
*fdRootsSocket = createUnixDomainSocket();
|
||||
try {
|
||||
nix::connect(fdRootsSocket->get(), socketPath);
|
||||
nix::connect(toSocket(fdRootsSocket->get()), socketPath);
|
||||
} catch (SysError & e) {
|
||||
/* The garbage collector may have exited or not
|
||||
created the socket yet, so we need to restart. */
|
||||
@ -166,12 +168,16 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor)
|
||||
// those to keep the directory alive.
|
||||
continue;
|
||||
}
|
||||
Path path = i.path();
|
||||
Path path = i.path().string();
|
||||
|
||||
pid_t pid = std::stoi(name);
|
||||
|
||||
debug("reading temporary root file '%1%'", path);
|
||||
AutoCloseFD fd(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666));
|
||||
AutoCloseFD fd(toDescriptor(open(path.c_str(),
|
||||
#ifndef _WIN32
|
||||
O_CLOEXEC |
|
||||
#endif
|
||||
O_RDWR, 0666)));
|
||||
if (!fd) {
|
||||
/* It's okay if the file has disappeared. */
|
||||
if (errno == ENOENT) continue;
|
||||
@ -240,8 +246,7 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R
|
||||
unlink(path.c_str());
|
||||
}
|
||||
} else {
|
||||
struct stat st2 = lstat(target);
|
||||
if (!S_ISLNK(st2.st_mode)) return;
|
||||
if (!std::filesystem::is_symlink(target)) return;
|
||||
Path target2 = readLink(target);
|
||||
if (isInStore(target2)) foundRoot(target, target2);
|
||||
}
|
||||
@ -297,24 +302,25 @@ Roots LocalStore::findRoots(bool censor)
|
||||
return roots;
|
||||
}
|
||||
|
||||
typedef std::unordered_map<Path, std::unordered_set<std::string>> UncheckedRoots;
|
||||
/**
|
||||
* Key is a mere string because cannot has path with macOS's libc++
|
||||
*/
|
||||
typedef std::unordered_map<std::string, std::unordered_set<std::string>> UncheckedRoots;
|
||||
|
||||
static void readProcLink(const std::string & file, UncheckedRoots & roots)
|
||||
static void readProcLink(const std::filesystem::path & file, UncheckedRoots & roots)
|
||||
{
|
||||
constexpr auto bufsiz = PATH_MAX;
|
||||
char buf[bufsiz];
|
||||
auto res = readlink(file.c_str(), buf, bufsiz);
|
||||
if (res == -1) {
|
||||
if (errno == ENOENT || errno == EACCES || errno == ESRCH)
|
||||
std::filesystem::path buf;
|
||||
try {
|
||||
buf = std::filesystem::read_symlink(file);
|
||||
} catch (std::filesystem::filesystem_error & e) {
|
||||
if (e.code() == std::errc::no_such_file_or_directory
|
||||
|| e.code() == std::errc::permission_denied
|
||||
|| e.code() == std::errc::no_such_process)
|
||||
return;
|
||||
throw SysError("reading symlink");
|
||||
throw;
|
||||
}
|
||||
if (res == bufsiz) {
|
||||
throw Error("overly long symlink starting with '%1%'", std::string_view(buf, bufsiz));
|
||||
}
|
||||
if (res > 0 && buf[0] == '/')
|
||||
roots[std::string(static_cast<char *>(buf), res)]
|
||||
.emplace(file);
|
||||
if (buf.is_absolute())
|
||||
roots[buf.string()].emplace(file.string());
|
||||
}
|
||||
|
||||
static std::string quoteRegexChars(const std::string & raw)
|
||||
@ -371,12 +377,12 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
|
||||
}
|
||||
fdDir.reset();
|
||||
|
||||
auto mapFile = fmt("/proc/%s/maps", ent->d_name);
|
||||
auto mapLines = tokenizeString<std::vector<std::string>>(readFile(mapFile), "\n");
|
||||
std::filesystem::path mapFile = fmt("/proc/%s/maps", ent->d_name);
|
||||
auto mapLines = tokenizeString<std::vector<std::string>>(readFile(mapFile.string()), "\n");
|
||||
for (const auto & line : mapLines) {
|
||||
auto match = std::smatch{};
|
||||
if (std::regex_match(line, match, mapRegex))
|
||||
unchecked[match[1]].emplace(mapFile);
|
||||
unchecked[match[1]].emplace(mapFile.string());
|
||||
}
|
||||
|
||||
auto envFile = fmt("/proc/%s/environ", ent->d_name);
|
||||
@ -407,7 +413,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
|
||||
for (const auto & line : lsofLines) {
|
||||
std::smatch match;
|
||||
if (std::regex_match(line, match, lsofRegex))
|
||||
unchecked[match[1]].emplace("{lsof}");
|
||||
unchecked[match[1].str()].emplace("{lsof}");
|
||||
}
|
||||
} catch (ExecError & e) {
|
||||
/* lsof not installed, lsof failed */
|
||||
@ -490,6 +496,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||
createDirs(dirOf(socketPath));
|
||||
auto fdServer = createUnixDomainSocket(socketPath, 0666);
|
||||
|
||||
// TODO nonblocking socket on windows?
|
||||
#ifdef _WIN32
|
||||
throw UnimplementedError("External GC client not implemented yet");
|
||||
#else
|
||||
if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) | O_NONBLOCK) == -1)
|
||||
throw SysError("making socket '%1%' non-blocking", socketPath);
|
||||
|
||||
@ -590,6 +600,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||
if (serverThread.joinable()) serverThread.join();
|
||||
});
|
||||
|
||||
#endif
|
||||
|
||||
/* Find the roots. Since we've grabbed the GC lock, the set of
|
||||
permanent roots cannot increase now. */
|
||||
printInfo("finding garbage collector roots...");
|
||||
@ -623,8 +635,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||
by another process. We need to be sure that we can acquire an
|
||||
exclusive lock before deleting them. */
|
||||
if (baseName.find("tmp-", 0) == 0) {
|
||||
AutoCloseFD tmpDirFd = open(realPath.c_str(), O_RDONLY | O_DIRECTORY);
|
||||
if (tmpDirFd.get() == -1 || !lockFile(tmpDirFd.get(), ltWrite, false)) {
|
||||
AutoCloseFD tmpDirFd = openDirectory(realPath);
|
||||
if (!tmpDirFd || !lockFile(tmpDirFd.get(), ltWrite, false)) {
|
||||
debug("skipping locked tempdir '%s'", realPath);
|
||||
return;
|
||||
}
|
||||
@ -857,7 +869,13 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||
struct stat st;
|
||||
if (stat(linksDir.c_str(), &st) == -1)
|
||||
throw SysError("statting '%1%'", linksDir);
|
||||
int64_t overhead = st.st_blocks * 512ULL;
|
||||
int64_t overhead =
|
||||
#ifdef _WIN32
|
||||
0
|
||||
#else
|
||||
st.st_blocks * 512ULL
|
||||
#endif
|
||||
;
|
||||
|
||||
printInfo("note: currently hard linking saves %.2f MiB",
|
||||
((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
|
||||
@ -870,6 +888,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||
|
||||
void LocalStore::autoGC(bool sync)
|
||||
{
|
||||
#ifdef HAVE_STATVFS
|
||||
static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE");
|
||||
|
||||
auto getAvail = [this]() -> uint64_t {
|
||||
@ -946,6 +965,7 @@ void LocalStore::autoGC(bool sync)
|
||||
sync:
|
||||
// Wait for the future outside of the state lock.
|
||||
if (sync) future.get();
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@ -424,8 +424,10 @@ public:
|
||||
Setting<bool> useSQLiteWAL{this, !isWSL1(), "use-sqlite-wal",
|
||||
"Whether SQLite should use WAL mode."};
|
||||
|
||||
#ifndef _WIN32
|
||||
Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering",
|
||||
"Whether to call `sync()` before registering a path as valid."};
|
||||
#endif
|
||||
|
||||
Setting<bool> useSubstitutes{
|
||||
this, true, "substitute",
|
||||
|
||||
@ -26,7 +26,6 @@
|
||||
#include <new>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/select.h>
|
||||
#include <sys/time.h>
|
||||
#include <unistd.h>
|
||||
#include <utime.h>
|
||||
@ -34,17 +33,20 @@
|
||||
#include <errno.h>
|
||||
#include <stdio.h>
|
||||
#include <time.h>
|
||||
#include <grp.h>
|
||||
|
||||
#ifndef _WIN32
|
||||
# include <grp.h>
|
||||
#endif
|
||||
|
||||
#if __linux__
|
||||
#include <sched.h>
|
||||
#include <sys/statvfs.h>
|
||||
#include <sys/mount.h>
|
||||
#include <sys/ioctl.h>
|
||||
# include <sched.h>
|
||||
# include <sys/statvfs.h>
|
||||
# include <sys/mount.h>
|
||||
# include <sys/ioctl.h>
|
||||
#endif
|
||||
|
||||
#ifdef __CYGWIN__
|
||||
#include <windows.h>
|
||||
# include <windows.h>
|
||||
#endif
|
||||
|
||||
#include <sqlite3.h>
|
||||
@ -52,8 +54,6 @@
|
||||
|
||||
namespace nix {
|
||||
|
||||
using namespace nix::unix;
|
||||
|
||||
std::string LocalStoreConfig::doc()
|
||||
{
|
||||
return
|
||||
@ -224,6 +224,7 @@ LocalStore::LocalStore(const Params & params)
|
||||
}
|
||||
}
|
||||
|
||||
#ifndef _WIN32
|
||||
/* Optionally, create directories and set permissions for a
|
||||
multi-user install. */
|
||||
if (isRootUser() && settings.buildUsersGroup != "") {
|
||||
@ -245,6 +246,7 @@ LocalStore::LocalStore(const Params & params)
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Ensure that the store and its parents are not symlinks. */
|
||||
if (!settings.allowSymlinkedStore) {
|
||||
@ -270,14 +272,25 @@ LocalStore::LocalStore(const Params & params)
|
||||
if (stat(reservedPath.c_str(), &st) == -1 ||
|
||||
st.st_size != settings.reservedSize)
|
||||
{
|
||||
AutoCloseFD fd = open(reservedPath.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0600);
|
||||
AutoCloseFD fd = toDescriptor(open(reservedPath.c_str(), O_WRONLY | O_CREAT
|
||||
#ifndef _WIN32
|
||||
| O_CLOEXEC
|
||||
#endif
|
||||
, 0600));
|
||||
int res = -1;
|
||||
#if HAVE_POSIX_FALLOCATE
|
||||
res = posix_fallocate(fd.get(), 0, settings.reservedSize);
|
||||
#endif
|
||||
if (res == -1) {
|
||||
writeFull(fd.get(), std::string(settings.reservedSize, 'X'));
|
||||
[[gnu::unused]] auto res2 = ftruncate(fd.get(), settings.reservedSize);
|
||||
[[gnu::unused]] auto res2 =
|
||||
|
||||
#ifdef _WIN32
|
||||
SetEndOfFile(fd.get())
|
||||
#else
|
||||
ftruncate(fd.get(), settings.reservedSize)
|
||||
#endif
|
||||
;
|
||||
}
|
||||
}
|
||||
} catch (SystemError & e) { /* don't care about errors */
|
||||
@ -460,10 +473,14 @@ LocalStore::LocalStore(std::string scheme, std::string path, const Params & para
|
||||
AutoCloseFD LocalStore::openGCLock()
|
||||
{
|
||||
Path fnGCLock = stateDir + "/gc.lock";
|
||||
auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
|
||||
auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT
|
||||
#ifndef _WIN32
|
||||
| O_CLOEXEC
|
||||
#endif
|
||||
, 0600);
|
||||
if (!fdGCLock)
|
||||
throw SysError("opening global GC lock '%1%'", fnGCLock);
|
||||
return fdGCLock;
|
||||
return toDescriptor(fdGCLock);
|
||||
}
|
||||
|
||||
|
||||
@ -491,7 +508,7 @@ LocalStore::~LocalStore()
|
||||
try {
|
||||
auto fdTempRoots(_fdTempRoots.lock());
|
||||
if (*fdTempRoots) {
|
||||
*fdTempRoots = -1;
|
||||
fdTempRoots->close();
|
||||
unlink(fnTempRoots.c_str());
|
||||
}
|
||||
} catch (...) {
|
||||
@ -969,11 +986,13 @@ void LocalStore::registerValidPath(const ValidPathInfo & info)
|
||||
|
||||
void LocalStore::registerValidPaths(const ValidPathInfos & infos)
|
||||
{
|
||||
#ifndef _WIN32
|
||||
/* SQLite will fsync by default, but the new valid paths may not
|
||||
be fsync-ed. So some may want to fsync them before registering
|
||||
the validity, at the expense of some speed of the path
|
||||
registering operation. */
|
||||
if (settings.syncBeforeRegistering) sync();
|
||||
#endif
|
||||
|
||||
return retrySQLite<void>([&]() {
|
||||
auto state(_state.lock());
|
||||
@ -1155,7 +1174,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
|
||||
|
||||
autoGC();
|
||||
|
||||
canonicalisePathMetaData(realPath, {});
|
||||
canonicalisePathMetaData(realPath);
|
||||
|
||||
optimisePath(realPath, repair); // FIXME: combine with hashPath()
|
||||
|
||||
@ -1307,7 +1326,7 @@ StorePath LocalStore::addToStoreFromDump(
|
||||
narHash = narSink.finish();
|
||||
}
|
||||
|
||||
canonicalisePathMetaData(realPath, {}); // FIXME: merge into restorePath
|
||||
canonicalisePathMetaData(realPath); // FIXME: merge into restorePath
|
||||
|
||||
optimisePath(realPath, repair);
|
||||
|
||||
@ -1340,8 +1359,8 @@ std::pair<std::filesystem::path, AutoCloseFD> LocalStore::createTempDirInStore()
|
||||
the GC between createTempDir() and when we acquire a lock on it.
|
||||
We'll repeat until 'tmpDir' exists and we've locked it. */
|
||||
tmpDirFn = createTempDir(realStoreDir, "tmp");
|
||||
tmpDirFd = open(tmpDirFn.c_str(), O_RDONLY | O_DIRECTORY);
|
||||
if (tmpDirFd.get() < 0) {
|
||||
tmpDirFd = openDirectory(tmpDirFn);
|
||||
if (!tmpDirFd) {
|
||||
continue;
|
||||
}
|
||||
lockedByUs = lockFile(tmpDirFd.get(), ltWrite, true);
|
||||
@ -1390,19 +1409,16 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
|
||||
for (auto & link : readDirectory(linksDir)) {
|
||||
auto name = link.path().filename();
|
||||
printMsg(lvlTalkative, "checking contents of '%s'", name);
|
||||
Path linkPath = linksDir / name;
|
||||
PosixSourceAccessor accessor;
|
||||
std::string hash = hashPath(
|
||||
{getFSSourceAccessor(), CanonPath(linkPath)},
|
||||
PosixSourceAccessor::createAtRoot(link.path()),
|
||||
FileIngestionMethod::Recursive, HashAlgorithm::SHA256).to_string(HashFormat::Nix32, false);
|
||||
if (hash != name.string()) {
|
||||
printError("link '%s' was modified! expected hash '%s', got '%s'",
|
||||
linkPath, name, hash);
|
||||
link.path(), name, hash);
|
||||
if (repair) {
|
||||
if (unlink(linkPath.c_str()) == 0)
|
||||
printInfo("removed link '%s'", linkPath);
|
||||
else
|
||||
throw SysError("removing corrupt link '%s'", linkPath);
|
||||
std::filesystem::remove(link.path());
|
||||
printInfo("removed link '%s'", link.path());
|
||||
} else {
|
||||
errors = true;
|
||||
}
|
||||
@ -1583,8 +1599,12 @@ static void makeMutable(const Path & path)
|
||||
/* The O_NOFOLLOW is important to prevent us from changing the
|
||||
mutable bit on the target of a symlink (which would be a
|
||||
security hole). */
|
||||
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
|
||||
if (fd == -1) {
|
||||
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW
|
||||
#ifndef _WIN32
|
||||
| O_CLOEXEC
|
||||
#endif
|
||||
);
|
||||
if (fd == INVALID_DESCRIPTOR) {
|
||||
if (errno == ELOOP) return; // it's a symlink
|
||||
throw SysError("opening file '%1%'", path);
|
||||
}
|
||||
@ -32,7 +32,6 @@ struct OptimiseStats
|
||||
{
|
||||
unsigned long filesLinked = 0;
|
||||
uint64_t bytesFreed = 0;
|
||||
uint64_t blocksFreed = 0;
|
||||
};
|
||||
|
||||
struct LocalStoreConfig : virtual LocalFSStoreConfig
|
||||
@ -89,11 +89,11 @@ else
|
||||
endif
|
||||
endif
|
||||
|
||||
$(d)/unix/local-store.cc: $(d)/unix/schema.sql.gen.hh $(d)/unix/ca-specific-schema.sql.gen.hh
|
||||
$(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
|
||||
|
||||
$(d)/unix/build.cc:
|
||||
|
||||
clean-files += $(d)/unix/schema.sql.gen.hh $(d)/unix/ca-specific-schema.sql.gen.hh
|
||||
clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
|
||||
|
||||
$(eval $(call install-file-in, $(buildprefix)$(d)/nix-store.pc, $(libdir)/pkgconfig, 0644))
|
||||
|
||||
|
||||
@ -155,55 +155,53 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
|
||||
debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true));
|
||||
|
||||
/* Check if this is a known hash. */
|
||||
Path linkPath = linksDir + "/" + hash.to_string(HashFormat::Nix32, false);
|
||||
std::filesystem::path linkPath = std::filesystem::path{linksDir} / hash.to_string(HashFormat::Nix32, false);
|
||||
|
||||
/* Maybe delete the link, if it has been corrupted. */
|
||||
if (pathExists(linkPath)) {
|
||||
auto stLink = lstat(linkPath);
|
||||
if (std::filesystem::exists(std::filesystem::symlink_status(linkPath))) {
|
||||
auto stLink = lstat(linkPath.string());
|
||||
if (st.st_size != stLink.st_size
|
||||
|| (repair && hash != ({
|
||||
hashPath(
|
||||
{make_ref<PosixSourceAccessor>(), CanonPath(linkPath)},
|
||||
PosixSourceAccessor::createAtRoot(linkPath),
|
||||
FileSerialisationMethod::Recursive, HashAlgorithm::SHA256).first;
|
||||
})))
|
||||
{
|
||||
// XXX: Consider overwriting linkPath with our valid version.
|
||||
warn("removing corrupted link '%s'", linkPath);
|
||||
warn("removing corrupted link %s", linkPath);
|
||||
warn("There may be more corrupted paths."
|
||||
"\nYou should run `nix-store --verify --check-contents --repair` to fix them all");
|
||||
unlink(linkPath.c_str());
|
||||
std::filesystem::remove(linkPath);
|
||||
}
|
||||
}
|
||||
|
||||
if (!pathExists(linkPath)) {
|
||||
if (!std::filesystem::exists(std::filesystem::symlink_status(linkPath))) {
|
||||
/* Nope, create a hard link in the links directory. */
|
||||
if (link(path.c_str(), linkPath.c_str()) == 0) {
|
||||
try {
|
||||
std::filesystem::create_hard_link(path, linkPath);
|
||||
inodeHash.insert(st.st_ino);
|
||||
return;
|
||||
}
|
||||
|
||||
switch (errno) {
|
||||
case EEXIST:
|
||||
} catch (std::filesystem::filesystem_error & e) {
|
||||
if (e.code() == std::errc::file_exists) {
|
||||
/* Fall through if another process created ‘linkPath’ before
|
||||
we did. */
|
||||
break;
|
||||
}
|
||||
|
||||
case ENOSPC:
|
||||
else if (e.code() == std::errc::no_space_on_device) {
|
||||
/* On ext4, that probably means the directory index is
|
||||
full. When that happens, it's fine to ignore it: we
|
||||
just effectively disable deduplication of this
|
||||
file. */
|
||||
printInfo("cannot link '%s' to '%s': %s", linkPath, path, strerror(errno));
|
||||
return;
|
||||
}
|
||||
|
||||
default:
|
||||
throw SysError("cannot link '%1%' to '%2%'", linkPath, path);
|
||||
else throw;
|
||||
}
|
||||
}
|
||||
|
||||
/* Yes! We've seen a file with the same contents. Replace the
|
||||
current file with a hard link to that file. */
|
||||
auto stLink = lstat(linkPath);
|
||||
auto stLink = lstat(linkPath.string());
|
||||
|
||||
if (st.st_ino == stLink.st_ino) {
|
||||
debug("'%1%' is already linked to '%2%'", path, linkPath);
|
||||
@ -223,10 +221,13 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
|
||||
its timestamp back to 0. */
|
||||
MakeReadOnly makeReadOnly(mustToggle ? dirOfPath : "");
|
||||
|
||||
Path tempLink = fmt("%1%/.tmp-link-%2%-%3%", realStoreDir, getpid(), random());
|
||||
std::filesystem::path tempLink = fmt("%1%/.tmp-link-%2%-%3%", realStoreDir, getpid(), rand());
|
||||
|
||||
if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
|
||||
if (errno == EMLINK) {
|
||||
try {
|
||||
std::filesystem::create_hard_link(linkPath, tempLink);
|
||||
inodeHash.insert(st.st_ino);
|
||||
} catch (std::filesystem::filesystem_error & e) {
|
||||
if (e.code() == std::errc::too_many_links) {
|
||||
/* Too many links to the same file (>= 32000 on most file
|
||||
systems). This is likely to happen with empty files.
|
||||
Just shrug and ignore. */
|
||||
@ -234,16 +235,16 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
|
||||
printInfo("'%1%' has maximum number of links", linkPath);
|
||||
return;
|
||||
}
|
||||
throw SysError("cannot link '%1%' to '%2%'", tempLink, linkPath);
|
||||
throw;
|
||||
}
|
||||
|
||||
/* Atomically replace the old file with the new hard link. */
|
||||
try {
|
||||
renameFile(tempLink, path);
|
||||
} catch (SystemError & e) {
|
||||
if (unlink(tempLink.c_str()) == -1)
|
||||
std::filesystem::rename(tempLink, path);
|
||||
} catch (std::filesystem::filesystem_error & e) {
|
||||
std::filesystem::remove(tempLink);
|
||||
printError("unable to unlink '%1%'", tempLink);
|
||||
if (errno == EMLINK) {
|
||||
if (e.code() == std::errc::too_many_links) {
|
||||
/* Some filesystems generate too many links on the rename,
|
||||
rather than on the original link. (Probably it
|
||||
temporarily increases the st_nlink field before
|
||||
@ -256,10 +257,13 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
|
||||
|
||||
stats.filesLinked++;
|
||||
stats.bytesFreed += st.st_size;
|
||||
stats.blocksFreed += st.st_blocks;
|
||||
|
||||
if (act)
|
||||
act->result(resFileLinked, st.st_size, st.st_blocks);
|
||||
act->result(resFileLinked, st.st_size
|
||||
#ifndef _WIN32
|
||||
, st.st_blocks
|
||||
#endif
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@ -31,6 +31,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct
|
||||
|
||||
}
|
||||
|
||||
#ifndef _WIN32 // TODO implement
|
||||
if (st.st_mtime != mtimeStore) {
|
||||
struct timeval times[2];
|
||||
times[0].tv_sec = st.st_atime;
|
||||
@ -46,6 +47,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct
|
||||
#endif
|
||||
throw SysError("changing modification time of '%1%'", path);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@ -57,7 +59,9 @@ void canonicaliseTimestampAndPermissions(const Path & path)
|
||||
|
||||
static void canonicalisePathMetaData_(
|
||||
const Path & path,
|
||||
#ifndef _WIN32
|
||||
std::optional<std::pair<uid_t, uid_t>> uidRange,
|
||||
#endif
|
||||
InodesSeen & inodesSeen)
|
||||
{
|
||||
checkInterrupt();
|
||||
@ -99,6 +103,7 @@ static void canonicalisePathMetaData_(
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef _WIN32
|
||||
/* Fail if the file is not owned by the build user. This prevents
|
||||
us from messing up the ownership/permissions of files
|
||||
hard-linked into the output (e.g. "ln /etc/shadow $out/foo").
|
||||
@ -112,11 +117,13 @@ static void canonicalisePathMetaData_(
|
||||
assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
|
||||
inodesSeen.insert(Inode(st.st_dev, st.st_ino));
|
||||
|
||||
canonicaliseTimestampAndPermissions(path, st);
|
||||
|
||||
#ifndef _WIN32
|
||||
/* Change ownership to the current uid. If it's a symlink, use
|
||||
lchown if available, otherwise don't bother. Wrong ownership
|
||||
of a symlink doesn't matter, since the owning user can't change
|
||||
@ -134,22 +141,36 @@ static void canonicalisePathMetaData_(
|
||||
throw SysError("changing owner of '%1%' to %2%",
|
||||
path, geteuid());
|
||||
}
|
||||
#endif
|
||||
|
||||
if (S_ISDIR(st.st_mode)) {
|
||||
std::vector<std::filesystem::directory_entry> entries = readDirectory(path);
|
||||
for (auto & i : entries)
|
||||
canonicalisePathMetaData_(i.path().string(), uidRange, inodesSeen);
|
||||
canonicalisePathMetaData_(
|
||||
i.path().string(),
|
||||
#ifndef _WIN32
|
||||
uidRange,
|
||||
#endif
|
||||
inodesSeen);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void canonicalisePathMetaData(
|
||||
const Path & path,
|
||||
#ifndef _WIN32
|
||||
std::optional<std::pair<uid_t, uid_t>> uidRange,
|
||||
#endif
|
||||
InodesSeen & inodesSeen)
|
||||
{
|
||||
canonicalisePathMetaData_(path, uidRange, inodesSeen);
|
||||
canonicalisePathMetaData_(
|
||||
path,
|
||||
#ifndef _WIN32
|
||||
uidRange,
|
||||
#endif
|
||||
inodesSeen);
|
||||
|
||||
#ifndef _WIN32
|
||||
/* On platforms that don't have lchown(), the top-level path can't
|
||||
be a symlink, since we can't change its ownership. */
|
||||
auto st = lstat(path);
|
||||
@ -158,14 +179,23 @@ void canonicalisePathMetaData(
|
||||
assert(S_ISLNK(st.st_mode));
|
||||
throw Error("wrong ownership of top-level store path '%1%'", path);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
void canonicalisePathMetaData(const Path & path,
|
||||
std::optional<std::pair<uid_t, uid_t>> uidRange)
|
||||
void canonicalisePathMetaData(const Path & path
|
||||
#ifndef _WIN32
|
||||
, std::optional<std::pair<uid_t, uid_t>> uidRange
|
||||
#endif
|
||||
)
|
||||
{
|
||||
InodesSeen inodesSeen;
|
||||
canonicalisePathMetaData(path, uidRange, inodesSeen);
|
||||
canonicalisePathMetaData_(
|
||||
path,
|
||||
#ifndef _WIN32
|
||||
uidRange,
|
||||
#endif
|
||||
inodesSeen);
|
||||
}
|
||||
|
||||
}
|
||||
@ -24,7 +24,7 @@ typedef std::set<Inode> InodesSeen;
|
||||
* without execute permission; setuid bits etc. are cleared)
|
||||
*
|
||||
* - the owner and group are set to the Nix user and group, if we're
|
||||
* running as root.
|
||||
* running as root. (Unix only.)
|
||||
*
|
||||
* If uidRange is not empty, this function will throw an error if it
|
||||
* encounters files owned by a user outside of the closed interval
|
||||
@ -32,11 +32,17 @@ typedef std::set<Inode> InodesSeen;
|
||||
*/
|
||||
void canonicalisePathMetaData(
|
||||
const Path & path,
|
||||
#ifndef _WIN32
|
||||
std::optional<std::pair<uid_t, uid_t>> uidRange,
|
||||
#endif
|
||||
InodesSeen & inodesSeen);
|
||||
|
||||
void canonicalisePathMetaData(
|
||||
const Path & path,
|
||||
std::optional<std::pair<uid_t, uid_t>> uidRange);
|
||||
const Path & path
|
||||
#ifndef _WIN32
|
||||
, std::optional<std::pair<uid_t, uid_t>> uidRange = std::nullopt
|
||||
#endif
|
||||
);
|
||||
|
||||
void canonicaliseTimestampAndPermissions(const Path & path);
|
||||
|
||||
@ -20,10 +20,6 @@
|
||||
#include "signals.hh"
|
||||
#include "users.hh"
|
||||
|
||||
#ifndef _WIN32
|
||||
# include "remote-store.hh"
|
||||
#endif
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <regex>
|
||||
|
||||
@ -1265,10 +1261,9 @@ Derivation Store::readInvalidDerivation(const StorePath & drvPath)
|
||||
|
||||
}
|
||||
|
||||
#ifndef _WIN32
|
||||
# include "local-store.hh"
|
||||
# include "uds-remote-store.hh"
|
||||
#endif
|
||||
|
||||
#include "local-store.hh"
|
||||
#include "uds-remote-store.hh"
|
||||
|
||||
|
||||
namespace nix {
|
||||
@ -1286,9 +1281,6 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_
|
||||
return {uri, params};
|
||||
}
|
||||
|
||||
#ifdef _WIN32 // Unused on Windows because the next `#ifndef`
|
||||
[[maybe_unused]]
|
||||
#endif
|
||||
static bool isNonUriPath(const std::string & spec)
|
||||
{
|
||||
return
|
||||
@ -1303,7 +1295,6 @@ std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Para
|
||||
{
|
||||
// TODO reenable on Windows once we have `LocalStore` and
|
||||
// `UDSRemoteStore`.
|
||||
#ifndef _WIN32
|
||||
if (uri == "" || uri == "auto") {
|
||||
auto stateDir = getOr(params, "state", settings.nixStateDir);
|
||||
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
|
||||
@ -1348,9 +1339,6 @@ std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Para
|
||||
} else {
|
||||
return nullptr;
|
||||
}
|
||||
#else
|
||||
return nullptr;
|
||||
#endif
|
||||
}
|
||||
|
||||
// The `parseURL` function supports both IPv6 URIs as defined in
|
||||
|
||||
@ -30,32 +30,6 @@
|
||||
#include <sys/utsname.h>
|
||||
#include <sys/resource.h>
|
||||
|
||||
#if HAVE_STATVFS
|
||||
#include <sys/statvfs.h>
|
||||
#endif
|
||||
|
||||
/* Includes required for chroot support. */
|
||||
#if __linux__
|
||||
#include <sys/socket.h>
|
||||
#include <sys/ioctl.h>
|
||||
#include <net/if.h>
|
||||
#include <netinet/ip.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sched.h>
|
||||
#include <sys/param.h>
|
||||
#include <sys/mount.h>
|
||||
#include <sys/syscall.h>
|
||||
#if HAVE_SECCOMP
|
||||
#include <seccomp.h>
|
||||
#endif
|
||||
#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
|
||||
#endif
|
||||
|
||||
#if __APPLE__
|
||||
#include <spawn.h>
|
||||
#include <sys/sysctl.h>
|
||||
#endif
|
||||
|
||||
#include <pwd.h>
|
||||
#include <grp.h>
|
||||
|
||||
|
||||
@ -94,7 +94,7 @@ Path canonPath(PathView path, bool resolveSymlinks)
|
||||
path,
|
||||
[&followCount, &temp, maxFollow, resolveSymlinks]
|
||||
(std::string & result, std::string_view & remaining) {
|
||||
if (resolveSymlinks && std::filesystem::is_symlink(result)) {
|
||||
if (resolveSymlinks && fs::is_symlink(result)) {
|
||||
if (++followCount >= maxFollow)
|
||||
throw Error("infinite symlink recursion in path '%0%'", remaining);
|
||||
remaining = (temp = concatStrings(readLink(result), remaining));
|
||||
|
||||
@ -117,6 +117,11 @@ bool pathAccessible(const Path & path);
|
||||
*/
|
||||
Path readLink(const Path & path);
|
||||
|
||||
/**
|
||||
* Open a `Descriptor` with read-only access to the given directory.
|
||||
*/
|
||||
Descriptor openDirectory(const std::filesystem::path & path);
|
||||
|
||||
/**
|
||||
* Read the contents of a directory. The entries `.` and `..` are
|
||||
* removed.
|
||||
|
||||
@ -39,6 +39,7 @@ using Socket =
|
||||
* Windows gives this a different name
|
||||
*/
|
||||
# define SHUT_WR SD_SEND
|
||||
# define SHUT_RDWR SD_BOTH
|
||||
#endif
|
||||
|
||||
/**
|
||||
|
||||
10
src/libutil/unix/file-system.cc
Normal file
10
src/libutil/unix/file-system.cc
Normal file
@ -0,0 +1,10 @@
|
||||
#include "file-system.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
Descriptor openDirectory(const std::filesystem::path & path)
|
||||
{
|
||||
return open(path.c_str(), O_RDONLY | O_DIRECTORY);
|
||||
}
|
||||
|
||||
}
|
||||
12
src/libutil/windows/file-system.cc
Normal file
12
src/libutil/windows/file-system.cc
Normal file
@ -0,0 +1,12 @@
|
||||
#include "file-system.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
Descriptor openDirectory(const std::filesystem::path & path)
|
||||
{
|
||||
return CreateFileW(
|
||||
path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, OPEN_EXISTING,
|
||||
FILE_FLAG_BACKUP_SEMANTICS, NULL);
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user