nix-copy-closure: Implement in C++.

Tests fail currently because the database is not given proper hashes in the VM
This commit is contained in:
Shea Levy 2016-08-10 10:44:39 -04:00
parent 8af062f372
commit bfa41eb671
11 changed files with 100 additions and 115 deletions

2
.gitignore vendored
View File

@ -81,6 +81,8 @@ Makefile.config
# /src/nix-build/ # /src/nix-build/
/src/nix-build/nix-build /src/nix-build/nix-build
/src/nix-copy-closure/nix-copy-closure
/src/build-remote/build-remote /src/build-remote/build-remote
# /tests/ # /tests/

View File

@ -11,6 +11,7 @@ makefiles = \
src/nix-env/local.mk \ src/nix-env/local.mk \
src/nix-daemon/local.mk \ src/nix-daemon/local.mk \
src/nix-collect-garbage/local.mk \ src/nix-collect-garbage/local.mk \
src/nix-copy-closure/local.mk \
src/nix-prefetch-url/local.mk \ src/nix-prefetch-url/local.mk \
src/buildenv/local.mk \ src/buildenv/local.mk \
src/resolve-system-dependencies/local.mk \ src/resolve-system-dependencies/local.mk \

View File

@ -1,8 +1,3 @@
nix_bin_scripts := \
$(d)/nix-copy-closure \
bin-scripts += $(nix_bin_scripts)
nix_noinst_scripts := \ nix_noinst_scripts := \
$(d)/build-remote.pl \ $(d)/build-remote.pl \
$(d)/nix-http-export.cgi \ $(d)/nix-http-export.cgi \
@ -16,4 +11,4 @@ profiledir = $(sysconfdir)/profile.d
$(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644)) $(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644))
$(eval $(call install-program-in, $(d)/build-remote.pl, $(libexecdir)/nix)) $(eval $(call install-program-in, $(d)/build-remote.pl, $(libexecdir)/nix))
clean-files += $(nix_bin_scripts) $(nix_noinst_scripts) clean-files += $(nix_noinst_scripts)

View File

@ -1,103 +0,0 @@
#! @perl@ -w @perlFlags@
use utf8;
use strict;
use Nix::SSH;
use Nix::Config;
use Nix::Store;
use Nix::CopyClosure;
use List::Util qw(sum);
binmode STDERR, ":encoding(utf8)";
if (scalar @ARGV < 1) {
print STDERR <<EOF
Usage: nix-copy-closure [--from | --to] HOSTNAME [--gzip] [--bzip2] [--xz] PATHS...
EOF
;
exit 1;
}
# Get the target host.
my $sshHost;
my $toMode = 1;
my $includeOutputs = 0;
my $dryRun = 0;
my $useSubstitutes = 0;
my $verbosity = 1;
# !!! Copied from nix-pack-closure, should put this in a module.
my @storePaths = ();
while (@ARGV) {
my $arg = shift @ARGV;
if ($arg eq "--help") {
exec "man nix-copy-closure" or die;
}
elsif ($arg eq "--gzip" || $arg eq "--bzip2" || $arg eq "--xz") {
warn "$0: $arg is not implemented\n" if $arg ne "--gzip";
push @globalSshOpts, "-C";
}
elsif ($arg eq "--from") {
$toMode = 0;
}
elsif ($arg eq "--to") {
$toMode = 1;
}
elsif ($arg eq "--include-outputs") {
$includeOutputs = 1;
}
elsif ($arg eq "--show-progress") {
warn "$0: $arg is not implemented\n";
}
elsif ($arg eq "--dry-run") {
$dryRun = 1;
}
elsif ($arg eq "--use-substitutes" || $arg eq "-s") {
$useSubstitutes = 1;
}
elsif ($arg eq "-v") {
$verbosity++;
setVerbosity($verbosity);
}
elsif (!defined $sshHost) {
$sshHost = $arg;
}
else {
push @storePaths, $arg;
}
}
die "$0: you did not specify a host name\n" unless defined $sshHost;
if ($toMode) { # Copy TO the remote machine.
Nix::CopyClosure::copyTo(
$sshHost, [ @storePaths ],
$includeOutputs, $dryRun, $useSubstitutes);
}
else { # Copy FROM the remote machine.
my ($from, $to) = connectToRemoteNix($sshHost, []);
# Query the closure of the given store paths on the remote
# machine. Paths are assumed to be store paths; there is no
# resolution (following of symlinks).
syswrite($to, pack("L<x4L<x4", 7, $includeOutputs ? 1 : 0)) or die;
writeStrings(\@storePaths, $to);
my @missing = grep { !isValidPath($_) } readStrings($from);
# Export the store paths on the remote machine and import them locally.
if (scalar @missing > 0) {
print STDERR "copying ", scalar @missing, " missing paths from $sshHost...\n";
writeInt(5, $to); # == cmdExportPaths
writeInt(0, $to); # obsolete
writeStrings(\@missing, $to);
importPaths(fileno($from), 1);
}
}

View File

@ -9,7 +9,6 @@
#include "shared.hh" #include "shared.hh"
#include "pathlocks.hh" #include "pathlocks.hh"
#include "globals.hh" #include "globals.hh"
#include "serve-protocol.hh"
#include "serialise.hh" #include "serialise.hh"
#include "store-api.hh" #include "store-api.hh"
#include "derivations.hh" #include "derivations.hh"

View File

@ -8,4 +8,4 @@ build-remote_LIBS = libmain libutil libformat libstore
build-remote_SOURCES := $(d)/build-remote.cc build-remote_SOURCES := $(d)/build-remote.cc
build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\"" -Isrc/nix-store build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\""

View File

@ -39,6 +39,8 @@ private:
string uri; string uri;
Path key; Path key;
bool compress;
}; };
SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections) SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections)
@ -48,6 +50,7 @@ SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections)
, socketPath((Path) tmpDir + "/ssh.sock") , socketPath((Path) tmpDir + "/ssh.sock")
, uri(std::move(uri)) , uri(std::move(uri))
, key(get(params, "ssh-key", "")) , key(get(params, "ssh-key", ""))
, compress(get(params, "compress", "") == "true")
{ {
/* open a connection and perform the handshake to verify all is well */ /* open a connection and perform the handshake to verify all is well */
connections->get(); connections->get();
@ -90,11 +93,12 @@ ref<FSAccessor> SSHStore::getFSAccessor()
ref<RemoteStore::Connection> SSHStore::openConnection() ref<RemoteStore::Connection> SSHStore::openConnection()
{ {
if ((pid_t) sshMaster == -1) { if ((pid_t) sshMaster == -1) {
auto flags = compress ? "-NMCS" : "-NMS";
sshMaster = startProcess([&]() { sshMaster = startProcess([&]() {
if (key.empty()) if (key.empty())
execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), uri.c_str(), NULL); execlp("ssh", "ssh", flags, socketPath.c_str(), uri.c_str(), NULL);
else else
execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), "-i", key.c_str(), uri.c_str(), NULL); execlp("ssh", "ssh", flags, socketPath.c_str(), "-i", key.c_str(), uri.c_str(), NULL);
throw SysError("starting ssh master"); throw SysError("starting ssh master");
}); });
} }

View File

@ -4,6 +4,7 @@
#include "util.hh" #include "util.hh"
#include "nar-info-disk-cache.hh" #include "nar-info-disk-cache.hh"
#include "thread-pool.hh" #include "thread-pool.hh"
#include "derivations.hh"
#include <future> #include <future>
@ -699,8 +700,27 @@ std::list<ref<Store>> getDefaultSubstituters()
} }
void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths) void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute)
{ {
if (substitute) {
/* Filter out .drv files (we don't want to build anything). */
PathSet paths2;
for (auto & path : storePaths)
if (!isDerivation(path)) paths2.insert(path);
unsigned long long downloadSize, narSize;
PathSet willBuild, willSubstitute, unknown;
to->queryMissing(PathSet(paths2.begin(), paths2.end()),
willBuild, willSubstitute, unknown, downloadSize, narSize);
/* FIXME: should use ensurePath(), but it only
does one path at a time. */
if (!willSubstitute.empty())
try {
to->buildPaths(willSubstitute);
} catch (Error & e) {
printMsg(lvlError, format("warning: %1%") % e.msg());
}
}
std::string copiedLabel = "copied"; std::string copiedLabel = "copied";
logger->setExpected(copiedLabel, storePaths.size()); logger->setExpected(copiedLabel, storePaths.size());

View File

@ -625,7 +625,7 @@ void removeTempRoots();
ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE")); ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"));
void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths); void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute = false);
enum StoreType { enum StoreType {
tDaemon, tDaemon,

View File

@ -0,0 +1,7 @@
programs += nix-copy-closure
nix-copy-closure_DIR := $(d)
nix-copy-closure_LIBS = libmain libutil libformat libstore
nix-copy-closure_SOURCES := $(d)/nix-copy-closure.cc

View File

@ -0,0 +1,60 @@
#include "shared.hh"
#include "store-api.hh"
using namespace nix;
int main(int argc, char ** argv)
{
return handleExceptions(argv[0], [&]() {
initNix();
auto gzip = false;
auto toMode = true;
auto includeOutputs = false;
auto dryRun = false;
auto useSubstitutes = false;
auto sshHost = string{};
auto storePaths = PathSet{};
parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
if (*arg == "--help")
showManPage("nix-copy-closure");
else if (*arg == "--version")
printVersion("nix-copy-closure");
else if (*arg == "--gzip" || *arg == "--bzip2" || *arg == "--xz") {
if (*arg != "--gzip")
printMsg(lvlError, format("Warning: %1% is not implemented, falling back to gzip") % *arg);
gzip = true;
} else if (*arg == "--from")
toMode = false;
else if (*arg == "--to")
toMode = true;
else if (*arg == "--include-outputs")
includeOutputs = true;
else if (*arg == "--show-progress")
printMsg(lvlError, "Warning: --show-progress is not implemented");
else if (*arg == "--dry-run")
dryRun = true;
else if (*arg == "--use-substitutes" || *arg == "-s")
useSubstitutes = true;
else if (sshHost.empty())
sshHost = *arg;
else
storePaths.insert(*arg);
return true;
});
if (sshHost.empty())
throw UsageError("no host name specified");
auto remoteUri = "ssh://" + sshHost + (gzip ? "?compress=true" : "");
auto to = toMode ? openStore(remoteUri) : openStore();
auto from = toMode ? openStore() : openStore(remoteUri);
if (includeOutputs) {
auto newPaths = PathSet{};
for (const auto & p : storePaths) {
auto outputs = from->queryDerivationOutputs(p);
newPaths.insert(outputs.begin(), outputs.end());
}
storePaths.insert(newPaths.begin(), newPaths.end());
}
copyPaths(from, to, Paths(storePaths.begin(), storePaths.end()), useSubstitutes);
});
}