Merge pull request #2748 from edolstra/rust

Make nix/unpack-channel.nix a builtin builder

.gitignore

@@ -85,6 +85,7 @@ perl/Makefile.config
 /tests/restricted-innocent
 /tests/shell
 /tests/shell.drv
+/tests/config.nix
 
 # /tests/lang/
 /tests/lang/*.out
@@ -118,3 +119,5 @@ GPATH
 GRTAGS
 GSYMS
 GTAGS
+
+nix-rust/target

Makefile

@@ -1,6 +1,7 @@
 makefiles = \
   mk/precompiled-headers.mk \
   local.mk \
+  nix-rust/local.mk \
   src/libutil/local.mk \
   src/libstore/local.mk \
   src/libmain/local.mk \

configure.ac

@@ -117,26 +117,15 @@ fi
 ])
 
 NEED_PROG(bash, bash)
-NEED_PROG(patch, patch)
 AC_PATH_PROG(xmllint, xmllint, false)
 AC_PATH_PROG(xsltproc, xsltproc, false)
 AC_PATH_PROG(flex, flex, false)
 AC_PATH_PROG(bison, bison, false)
-NEED_PROG(sed, sed)
-NEED_PROG(tar, tar)
-NEED_PROG(bzip2, bzip2)
-NEED_PROG(gzip, gzip)
-NEED_PROG(xz, xz)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(lsof, lsof, lsof)
 
 
-NEED_PROG(cat, cat)
-NEED_PROG(tr, tr)
-AC_ARG_WITH(coreutils-bin, AC_HELP_STRING([--with-coreutils-bin=PATH],
-  [path of cat, mkdir, etc.]),
-  coreutils=$withval, coreutils=$(dirname $cat))
-AC_SUBST(coreutils)
+AC_SUBST(coreutils, [$(dirname $(type -p cat))])
 
 
 AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH],
@@ -167,7 +156,8 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then
     LIBS="-latomic $LIBS"
 fi
 
-# Look for OpenSSL, a required dependency.
+# Look for OpenSSL, a required dependency. FIXME: this is only (maybe)
+# used by S3BinaryCacheStore.
 PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])
 
 
@@ -177,11 +167,9 @@ AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true],
 AC_CHECK_HEADERS([bzlib.h], [true],
   [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
 
-
 # Look for SQLite, a required dependency.
 PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"])
 
-
 # Look for libcurl, a required dependency.
 PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"])
 
@@ -204,13 +192,11 @@ PKG_CHECK_MODULES([SODIUM], [libsodium],
    have_sodium=1], [have_sodium=])
 AC_SUBST(HAVE_SODIUM, [$have_sodium])
 
-
 # Look for liblzma, a required dependency.
 PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])
 AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt],
   [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])])
 
-
 # Look for libbrotli{enc,dec}.
 PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"])
 

corepkgs/config.nix.in

@@ -1,29 +1,13 @@
+# FIXME: remove this file?
 let
   fromEnv = var: def:
     let val = builtins.getEnv var; in
     if val != "" then val else def;
 in rec {
-  shell = "@bash@";
-  coreutils = "@coreutils@";
-  bzip2 = "@bzip2@";
-  gzip = "@gzip@";
-  xz = "@xz@";
-  tar = "@tar@";
-  tarFlags = "@tarFlags@";
-  tr = "@tr@";
   nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@";
   nixPrefix = "@prefix@";
   nixLibexecDir = fromEnv "NIX_LIBEXEC_DIR" "@libexecdir@";
   nixLocalstateDir = "@localstatedir@";
   nixSysconfDir = "@sysconfdir@";
   nixStoreDir = fromEnv "NIX_STORE_DIR" "@storedir@";
-
-  # If Nix is installed in the Nix store, then automatically add it as
-  # a dependency to the core packages. This ensures that they work
-  # properly in a chroot.
-  chrootDeps =
-    if dirOf nixPrefix == builtins.storeDir then
-      [ (builtins.storePath nixPrefix) ]
-    else
-      [ ];
 }

corepkgs/unpack-channel.nix

@@ -1,39 +1,12 @@
-with import <nix/config.nix>;
-
-let
-
-  builder = builtins.toFile "unpack-channel.sh"
-    ''
-      mkdir $out
-      cd $out
-      xzpat="\.xz\$"
-      gzpat="\.gz\$"
-      if [[ "$src" =~ $xzpat ]]; then
-        ${xz} -d < $src | ${tar} xf - ${tarFlags}
-      elif [[ "$src" =~ $gzpat ]]; then
-        ${gzip} -d < $src | ${tar} xf - ${tarFlags}
-      else
-        ${bzip2} -d < $src | ${tar} xf - ${tarFlags}
-      fi
-      if [ * != $channelName ]; then
-        mv * $out/$channelName
-      fi
-    '';
-
-in
-
 { name, channelName, src }:
 
 derivation {
-  system = builtins.currentSystem;
-  builder = shell;
-  args = [ "-e" builder ];
-  inherit name channelName src;
+  builder = "builtin:unpack-channel";
 
-  PATH = "${nixBinDir}:${coreutils}";
+  system = "builtin";
+
+  inherit name channelName src;
 
   # No point in doing this remotely.
   preferLocalBuild = true;
-
-  inherit chrootDeps;
 }

nix-rust/Cargo.lock

@@ -0,0 +1,84 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+[[package]]
+name = "cfg-if"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "filetime"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
+ "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.65"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "nix-rust"
+version = "0.1.0"
+dependencies = [
+ "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.1.56"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "tar"
+version = "0.4.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
+ "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
+ "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "winapi"
+version = "0.3.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "xattr"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[metadata]
+"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
+"checksum filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1ff6d4dab0aa0c8e6346d46052e93b13a16cf847b54ed357087c35011048cc7d"
+"checksum libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)" = "1a31a0627fdf1f6a39ec0dd577e101440b7db22672c0901fe00a9a6fbb5c24e8"
+"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
+"checksum tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)" = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3"
+"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6"
+"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+"checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c"

nix-rust/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "nix-rust"
+version = "0.1.0"
+authors = ["Eelco Dolstra <edolstra@gmail.com>"]
+edition = "2018"
+
+[lib]
+name = "nixrust"
+crate-type = ["cdylib"]
+
+[dependencies]
+tar = "0.4"
+libc = "0.2"

nix-rust/local.mk

@@ -0,0 +1,38 @@
+ifeq ($(OPTIMIZE), 1)
+  RUST_MODE = --release
+  RUST_DIR = release
+else
+  RUST_MODE =
+  RUST_DIR = debug
+endif
+
+libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.$(SO_EXT)
+libnixrust_INSTALL_PATH := $(libdir)/libnixrust.$(SO_EXT)
+libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl
+libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust -ldl
+
+ifeq ($(OS), Darwin)
+libnixrust_BUILD_FLAGS = NIX_LDFLAGS="-undefined dynamic_lookup"
+else
+libnixrust_LDFLAGS_USE += -Wl,-rpath,$(abspath $(d)/target/$(RUST_DIR))
+libnixrust_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$(libdir)
+endif
+
+$(libnixrust_PATH): $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml
+	$(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) \
+	$(libnixrust_BUILD_FLAGS) \
+	  cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \
+	&& touch target/$(RUST_DIR)/libnixrust.$(SO_EXT)
+
+$(libnixrust_INSTALL_PATH): $(libnixrust_PATH)
+	$(target-gen) cp $^ $@
+ifeq ($(OS), Darwin)
+	install_name_tool -id $@ $@
+endif
+
+dist-files += $(d)/vendor
+
+clean: clean-rust
+
+clean-rust:
+	$(suppress) rm -rfv nix-rust/target

nix-rust/src/error.rs

@@ -0,0 +1,31 @@
+#[derive(Debug)]
+pub enum Error {
+    IOError(std::io::Error),
+    Misc(String),
+    Foreign(CppException),
+}
+
+impl From<std::io::Error> for Error {
+    fn from(err: std::io::Error) -> Self {
+        Error::IOError(err)
+    }
+}
+
+impl From<Error> for CppException {
+    fn from(err: Error) -> Self {
+        match err {
+            Error::Foreign(ex) => ex,
+            Error::Misc(s) => unsafe { make_error(&s) },
+            Error::IOError(err) => unsafe { make_error(&err.to_string()) },
+        }
+    }
+}
+
+#[repr(C)]
+#[derive(Debug)]
+pub struct CppException(*const libc::c_void); // == std::exception_ptr*
+
+extern "C" {
+    #[allow(improper_ctypes)] // YOLO
+    fn make_error(s: &str) -> CppException;
+}

nix-rust/src/foreign.rs

@@ -0,0 +1,14 @@
+/// A wrapper around Nix's Source class that provides the Read trait.
+#[repr(C)]
+pub struct Source {
+    fun: extern "C" fn(this: *mut libc::c_void, data: &mut [u8]) -> usize,
+    this: *mut libc::c_void,
+}
+
+impl std::io::Read for Source {
+    fn read(&mut self, buf: &mut [u8]) -> std::result::Result<usize, std::io::Error> {
+        let n = (self.fun)(self.this, buf);
+        assert!(n <= buf.len());
+        Ok(n)
+    }
+}

nix-rust/src/lib.rs

@@ -0,0 +1,32 @@
+mod error;
+mod foreign;
+mod tarfile;
+
+pub use error::Error;
+
+pub struct CBox<T> {
+    pub ptr: *mut libc::c_void,
+    phantom: std::marker::PhantomData<T>,
+}
+
+impl<T> CBox<T> {
+    fn new(t: T) -> Self {
+        unsafe {
+            let size = std::mem::size_of::<T>();
+            let ptr = libc::malloc(size);
+            *(ptr as *mut T) = t; // FIXME: probably UB
+            Self {
+                ptr,
+                phantom: std::marker::PhantomData,
+            }
+        }
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn unpack_tarfile(
+    source: foreign::Source,
+    dest_dir: &str,
+) -> CBox<Result<(), error::CppException>> {
+    CBox::new(tarfile::unpack_tarfile(source, dest_dir).map_err(|err| err.into()))
+}

nix-rust/src/tarfile.rs

@@ -0,0 +1,46 @@
+use crate::{foreign::Source, Error};
+use std::fs;
+use std::io;
+use std::os::unix::fs::OpenOptionsExt;
+use std::path::Path;
+use tar::Archive;
+
+pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> {
+    let dest_dir = Path::new(dest_dir);
+
+    let mut tar = Archive::new(source);
+
+    for file in tar.entries()? {
+        let mut file = file?;
+
+        let dest_file = dest_dir.join(file.path()?);
+
+        fs::create_dir_all(dest_file.parent().unwrap())?;
+
+        match file.header().entry_type() {
+            tar::EntryType::Directory => {
+                fs::create_dir(dest_file)?;
+            }
+            tar::EntryType::Regular => {
+                let mode = if file.header().mode()? & (libc::S_IXUSR as u32) == 0 {
+                    0o666
+                } else {
+                    0o777
+                };
+                let mut f = fs::OpenOptions::new()
+                    .create(true)
+                    .write(true)
+                    .mode(mode)
+                    .open(dest_file)?;
+                io::copy(&mut file, &mut f)?;
+            }
+            tar::EntryType::Symlink => {
+                std::os::unix::fs::symlink(file.header().link_name()?.unwrap(), dest_file)?;
+            }
+            tar::EntryType::XGlobalHeader | tar::EntryType::XHeader => {}
+            t => return Err(Error::Misc(format!("unsupported tar entry type '{:?}'", t))),
+        }
+    }
+
+    Ok(())
+}

perl/lib/Nix/Config.pm.in

@@ -11,10 +11,6 @@ $logDir = $ENV{"NIX_LOG_DIR"} || "@nixlocalstatedir@/log/nix";
 $confDir = $ENV{"NIX_CONF_DIR"} || "@nixsysconfdir@/nix";
 $storeDir = $ENV{"NIX_STORE_DIR"} || "@nixstoredir@";
 
-$bzip2 = "@bzip2@";
-$xz = "@xz@";
-$curl = "@curl@";
-
 $useBindings = 1;
 
 %config = ();

release-common.nix

@@ -51,6 +51,7 @@ rec {
       openssl pkgconfig sqlite boehmgc
       boost
       nlohmann_json
+      rustc cargo
 
       # Tests
       git

release.nix

@@ -10,6 +10,50 @@ let
 
   jobs = rec {
 
+    # Create a "vendor" directory that contains the crates listed in
+    # Cargo.lock, and include it in the Nix tarball. This allows Nix
+    # to be built without network access.
+    vendoredCrates =
+      let
+        lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock);
+
+        files = map (pkg: import <nix/fetchurl.nix> {
+          url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download";
+          sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)";
+        }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package);
+
+      in pkgs.runCommand "cargo-vendor-dir" {}
+        ''
+          mkdir -p $out/vendor
+
+          cat > $out/vendor/config <<EOF
+          [source.crates-io]
+          replace-with = "vendored-sources"
+
+          [source.vendored-sources]
+          directory = "vendor"
+          EOF
+
+          ${toString (builtins.map (file: ''
+            mkdir $out/vendor/tmp
+            tar xvf ${file} -C $out/vendor/tmp
+            dir=$(echo $out/vendor/tmp/*)
+
+            # Add just enough metadata to keep Cargo happy.
+            printf '{"files":{},"package":"${file.outputHash}"}' > "$dir/.cargo-checksum.json"
+
+            # Clean up some cruft from the winapi crates. FIXME: find
+            # a way to remove winapi* from our dependencies.
+            if [[ $dir =~ /winapi ]]; then
+              find $dir -name "*.a" -print0 | xargs -0 rm -f --
+            fi
+
+            mv "$dir" $out/vendor/
+
+            rm -rf $out/vendor/tmp
+          '') files)}
+        '';
+
 
     tarball =
       with pkgs;
@@ -38,6 +82,8 @@ let
 
         distPhase =
           ''
+            cp -prd ${vendoredCrates}/vendor/ nix-rust/vendor/
+
             runHook preDist
             make dist
             mkdir -p $out/tarballs

shell.nix

@@ -7,7 +7,7 @@ with import ./release-common.nix { inherit pkgs; };
 (if useClang then clangStdenv else stdenv).mkDerivation {
   name = "nix";
 
-  buildInputs = buildDeps ++ tarballDeps ++ perlDeps;
+  buildInputs = buildDeps ++ tarballDeps ++ perlDeps ++ [ pkgs.rustfmt ];
 
   inherit configureFlags;
 

src/libexpr/primops/fetchGit.cc

@@ -4,6 +4,7 @@
 #include "store-api.hh"
 #include "pathlocks.hh"
 #include "hash.hh"
+#include "tarfile.hh"
 
 #include <sys/time.h>
 
@@ -164,14 +165,16 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
         if (e.errNo != ENOENT) throw;
     }
 
-    // FIXME: should pipe this, or find some better way to extract a
-    // revision.
-    auto tar = runProgram("git", true, { "-C", cacheDir, "archive", gitInfo.rev });
+    auto source = sinkToSource([&](Sink & sink) {
+        RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev });
+        gitOptions.standardOut = &sink;
+        runProgram2(gitOptions);
+    });
 
     Path tmpDir = createTempDir();
     AutoDelete delTmpDir(tmpDir, true);
 
-    runProgram("tar", true, { "x", "-C", tmpDir }, tar);
+    unpackTarfile(*source, tmpDir);
 
     gitInfo.storePath = store->addToStore(name, tmpDir);
 

src/libstore/build.cc

@@ -3128,6 +3128,8 @@ void DerivationGoal::runChild()
                     builtinFetchurl(drv2, netrcData);
                 else if (drv->builder == "builtin:buildenv")
                     builtinBuildenv(drv2);
+                else if (drv->builder == "builtin:unpack-channel")
+                    builtinUnpackChannel(drv2);
                 else
                     throw Error(format("unsupported builtin function '%1%'") % string(drv->builder, 8));
                 _exit(0);

src/libstore/builtins.hh

@@ -7,5 +7,6 @@ namespace nix {
 // TODO: make pluggable.
 void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData);
 void builtinBuildenv(const BasicDerivation & drv);
+void builtinUnpackChannel(const BasicDerivation & drv);
 
 }

src/libstore/builtins/unpack-channel.cc

@@ -0,0 +1,29 @@
+#include "builtins.hh"
+#include "tarfile.hh"
+
+namespace nix {
+
+void builtinUnpackChannel(const BasicDerivation & drv)
+{
+    auto getAttr = [&](const string & name) {
+        auto i = drv.env.find(name);
+        if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
+        return i->second;
+    };
+
+    Path out = getAttr("out");
+    auto channelName = getAttr("channelName");
+    auto src = getAttr("src");
+
+    createDirs(out);
+
+    unpackTarfile(src, out);
+
+    auto entries = readDirectory(out);
+    if (entries.size() != 1)
+        throw Error("channel tarball '%s' contains more than one file", src);
+    if (rename((out + "/" + entries[0].name).c_str(), (out + "/" + channelName).c_str()) == -1)
+        throw SysError("renaming channel directory");
+}
+
+}

src/libstore/download.cc

@@ -8,6 +8,7 @@
 #include "compression.hh"
 #include "pathlocks.hh"
 #include "finally.hh"
+#include "tarfile.hh"
 
 #ifdef ENABLE_S3
 #include <aws/core/client/ClientConfiguration.h>
@@ -903,12 +904,15 @@ CachedDownloadResult Downloader::downloadCached(
                 unpackedStorePath = "";
         }
         if (unpackedStorePath.empty()) {
-            printInfo(format("unpacking '%1%'...") % url);
+            printInfo("unpacking '%s'...", url);
             Path tmpDir = createTempDir();
             AutoDelete autoDelete(tmpDir, true);
-            // FIXME: this requires GNU tar for decompression.
-            runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"});
-            unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair);
+            unpackTarfile(store->toRealPath(storePath), tmpDir, baseNameOf(url));
+            auto members = readDirectory(tmpDir);
+            if (members.size() != 1)
+                throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
+            auto topDir = tmpDir + "/" + members.begin()->name;
+            unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair);
         }
         replaceSymlink(unpackedStorePath, unpackedLink);
         storePath = unpackedStorePath;

src/libutil/local.mk

@@ -7,3 +7,5 @@ libutil_DIR := $(d)
 libutil_SOURCES := $(wildcard $(d)/*.cc)
 
 libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(BOOST_LDFLAGS) -lboost_context
+
+libutil_LIBS = libnixrust

src/libutil/rust-ffi.cc

@@ -0,0 +1,12 @@
+#include "logging.hh"
+#include "rust-ffi.hh"
+
+namespace nix {
+
+extern "C" std::exception_ptr * make_error(rust::StringSlice s)
+{
+    // FIXME: leak
+    return new std::exception_ptr(std::make_exception_ptr(Error(std::string(s.ptr, s.size))));
+}
+
+}

src/libutil/rust-ffi.hh

@@ -0,0 +1,84 @@
+#include "serialise.hh"
+
+namespace rust {
+
+// Depending on the internal representation of Rust slices is slightly
+// evil...
+template<typename T>
+struct Slice
+{
+    T * ptr;
+    size_t size;
+
+    Slice(T * ptr, size_t size) : ptr(ptr), size(size)
+    {
+        assert(ptr);
+    }
+};
+
+struct StringSlice : Slice<char>
+{
+    StringSlice(const std::string & s): Slice((char *) s.data(), s.size()) {}
+};
+
+struct Source
+{
+    size_t (*fun)(void * source_this, rust::Slice<uint8_t> data);
+    nix::Source * _this;
+
+    Source(nix::Source & _this)
+        : fun(sourceWrapper), _this(&_this)
+    {}
+
+    // FIXME: how to propagate exceptions?
+    static size_t sourceWrapper(void * _this, rust::Slice<uint8_t> data)
+    {
+        auto n = ((nix::Source *) _this)->read(data.ptr, data.size);
+        return n;
+    }
+};
+
+/* C++ representation of Rust's Result<T, CppException>. */
+template<typename T>
+struct Result
+{
+    unsigned int tag;
+
+    union {
+        T data;
+        std::exception_ptr * exc;
+    };
+
+    /* Rethrow the wrapped exception or return the wrapped value. */
+    T unwrap()
+    {
+        if (tag == 0)
+            return data;
+        else if (tag == 1)
+            std::rethrow_exception(*exc);
+        else
+            abort();
+    }
+};
+
+template<typename T>
+struct CBox
+{
+    T * ptr;
+
+    T * operator ->()
+    {
+        return ptr;
+    }
+
+    CBox(T * ptr) : ptr(ptr) { }
+    CBox(const CBox &) = delete;
+    CBox(CBox &&) = delete;
+
+    ~CBox()
+    {
+        free(ptr);
+    }
+};
+
+}

src/libutil/serialise.hh

@@ -77,7 +77,6 @@ struct BufferedSource : Source
 
     size_t read(unsigned char * data, size_t len) override;
 
-
     bool hasData();
 
 protected:

src/libutil/tarfile.cc

@@ -0,0 +1,36 @@
+#include "rust-ffi.hh"
+#include "compression.hh"
+
+extern "C" {
+    rust::Result<std::tuple<>> *
+    unpack_tarfile(rust::Source source, rust::StringSlice dest_dir);
+}
+
+namespace nix {
+
+void unpackTarfile(Source & source, const Path & destDir)
+{
+    rust::Source source2(source);
+    rust::CBox(unpack_tarfile(source2, destDir))->unwrap();
+}
+
+void unpackTarfile(const Path & tarFile, const Path & destDir,
+    std::optional<std::string> baseName)
+{
+    if (!baseName) baseName = baseNameOf(tarFile);
+
+    auto source = sinkToSource([&](Sink & sink) {
+        // FIXME: look at first few bytes to determine compression type.
+        auto decompressor =
+            // FIXME: add .gz support
+            hasSuffix(*baseName, ".bz2") ? makeDecompressionSink("bzip2", sink) :
+            hasSuffix(*baseName, ".xz") ? makeDecompressionSink("xz", sink) :
+            makeDecompressionSink("none", sink);
+        readFile(tarFile, *decompressor);
+        decompressor->finish();
+    });
+
+    unpackTarfile(*source, destDir);
+}
+
+}

src/libutil/tarfile.hh

@@ -0,0 +1,10 @@
+#include "serialise.hh"
+
+namespace nix {
+
+void unpackTarfile(Source & source, const Path & destDir);
+
+void unpackTarfile(const Path & tarFile, const Path & destDir,
+    std::optional<std::string> baseName = {});
+
+}

src/nix-prefetch-url/nix-prefetch-url.cc

@@ -9,6 +9,7 @@
 #include "legacy.hh"
 #include "finally.hh"
 #include "progress-bar.hh"
+#include "tarfile.hh"
 
 #include <iostream>
 
@@ -192,8 +193,7 @@ static int _main(int argc, char * * argv)
                 if (hasSuffix(baseNameOf(uri), ".zip"))
                     runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked});
                 else
-                    // FIXME: this requires GNU tar for decompression.
-                    runProgram("tar", true, {"xf", tmpFile, "-C", unpacked});
+                    unpackTarfile(tmpFile, unpacked, baseNameOf(uri));
 
                 /* If the archive unpacks to a single file/directory, then use
                    that as the top-level. */

tests/config.nix.in

@@ -1,9 +1,7 @@
-with import <nix/config.nix>;
-
 rec {
-  inherit shell;
+  shell = "@bash@";
 
-  path = coreutils;
+  path = "@coreutils@";
 
   system = builtins.currentSystem;
 

tests/local.mk

@@ -39,4 +39,4 @@ tests-environment = NIX_REMOTE= $(bash) -e
 
 clean-files += $(d)/common.sh
 
-installcheck: $(d)/common.sh $(d)/plugins/libplugintest.$(SO_EXT)
+installcheck: $(d)/common.sh $(d)/config.nix $(d)/plugins/libplugintest.$(SO_EXT)