Refactor nixos-install to separate out filesystem build logic

The key distinction I'm drawing is that there's a component that deals
with the store of the machine being built, and another component for
the store building it. The inner part of it assumes nothing from the
builder (doesn't need chroot or root powers) so it can run comfortably
inside a Nix build, as well as nixos-rebuild. I have some upcoming work
that will use that to significantly speed up and streamline image builds
for NixOS, especially on virtualized hosts like EC2, but it's also a
reasonable speedup on native hosts.
This commit is contained in:
Dan Peebles 2017-02-20 19:57:16 +00:00
parent 91d2dc00bb
commit d990aa7163
5 changed files with 164 additions and 130 deletions

View File

@ -87,38 +87,6 @@ if ! test -e "$mountPoint"; then
exit 1
fi
# Mount some stuff in the target root directory.
mkdir -m 0755 -p $mountPoint/dev $mountPoint/proc $mountPoint/sys $mountPoint/etc $mountPoint/run $mountPoint/home
mkdir -m 01777 -p $mountPoint/tmp
mkdir -m 0755 -p $mountPoint/tmp/root
mkdir -m 0755 -p $mountPoint/var
mkdir -m 0700 -p $mountPoint/root
mount --rbind /dev $mountPoint/dev
mount --rbind /proc $mountPoint/proc
mount --rbind /sys $mountPoint/sys
mount --rbind / $mountPoint/tmp/root
mount -t tmpfs -o "mode=0755" none $mountPoint/run
rm -rf $mountPoint/var/run
ln -s /run $mountPoint/var/run
for f in /etc/resolv.conf /etc/hosts; do rm -f $mountPoint/$f; [ -f "$f" ] && cp -Lf $f $mountPoint/etc/; done
for f in /etc/passwd /etc/group; do touch $mountPoint/$f; [ -f "$f" ] && mount --rbind -o ro $f $mountPoint/$f; done
cp -Lf "@cacert@" "$mountPoint/tmp/ca-cert.crt"
export SSL_CERT_FILE=/tmp/ca-cert.crt
# For Nix 1.7
export CURL_CA_BUNDLE=/tmp/ca-cert.crt
if [ -n "$runChroot" ]; then
if ! [ -L $mountPoint/nix/var/nix/profiles/system ]; then
echo "$0: installation not finished; cannot chroot into installation directory"
exit 1
fi
ln -s /nix/var/nix/profiles/system $mountPoint/run/current-system
exec chroot $mountPoint "${chrootCommand[@]}"
fi
# Get the path of the NixOS configuration file.
if test -z "$NIXOS_CONFIG"; then
NIXOS_CONFIG=/etc/nixos/configuration.nix
@ -130,121 +98,60 @@ if [ ! -e "$mountPoint/$NIXOS_CONFIG" ] && [ -z "$closure" ]; then
fi
# Create the necessary Nix directories on the target device, if they
# don't already exist.
mkdir -m 0755 -p \
$mountPoint/nix/var/nix/gcroots \
$mountPoint/nix/var/nix/temproots \
$mountPoint/nix/var/nix/userpool \
$mountPoint/nix/var/nix/profiles \
$mountPoint/nix/var/nix/db \
$mountPoint/nix/var/log/nix/drvs
mkdir -m 1775 -p $mountPoint/nix/store
chown @root_uid@:@nixbld_gid@ $mountPoint/nix/store
# There is no daemon in the chroot.
unset NIX_REMOTE
# We don't have locale-archive in the chroot, so clear $LANG.
export LANG=
export LC_ALL=
export LC_TIME=
# Builds will use users that are members of this group
extraBuildFlags+=(--option "build-users-group" "$buildUsersGroup")
# Inherit binary caches from the host
# TODO: will this still work with Nix 1.12 now that it has no perl? Probably not...
binary_caches="$(@perl@/bin/perl -I @nix@/lib/perl5/site_perl/*/* -e 'use Nix::Config; Nix::Config::readConfig; print $Nix::Config::config{"binary-caches"};')"
extraBuildFlags+=(--option "binary-caches" "$binary_caches")
nixpkgs="$(readlink -f "$(nix-instantiate --find-file nixpkgs)")"
export NIX_PATH="nixpkgs=$nixpkgs:nixos-config=$mountPoint/$NIXOS_CONFIG"
unset NIXOS_CONFIG
# Copy Nix to the Nix store on the target device, unless it's already there.
if ! NIX_DB_DIR=$mountPoint/nix/var/nix/db nix-store --check-validity @nix@ 2> /dev/null; then
echo "copying Nix to $mountPoint...."
for i in $(@perl@/bin/perl @pathsFromGraph@ @nixClosure@); do
echo " $i"
chattr -R -i $mountPoint/$i 2> /dev/null || true # clear immutable bit
@rsync@/bin/rsync -a $i $mountPoint/nix/store/
done
# Register the paths in the Nix closure as valid. This is necessary
# to prevent them from being deleted the first time we install
# something. (I.e., Nix will see that, e.g., the glibc path is not
# valid, delete it to get it out of the way, but as a result nothing
# will work anymore.)
chroot $mountPoint @nix@/bin/nix-store --register-validity < @nixClosure@
fi
# TODO: do I need to set NIX_SUBSTITUTERS here or is the --option binary-caches above enough?
# Create the required /bin/sh symlink; otherwise lots of things
# (notably the system() function) won't work.
mkdir -m 0755 -p $mountPoint/bin
# !!! assuming that @shell@ is in the closure
ln -sf @shell@ $mountPoint/bin/sh
# A place to drop temporary closures
trap "rm -rf $tmpdir" EXIT
tmpdir="$(mktemp -d)"
# Build a closure (on the host; we then copy it into the guest)
function closure() {
nix-build "${extraBuildFlags[@]}" --no-out-link -E "with import <nixpkgs> {}; runCommand \"closure\" { exportReferencesGraph = [ \"x\" (buildEnv { name = \"env\"; paths = [ ($1) stdenv ]; }) ]; } \"cp x \$out\""
}
# Build hooks likely won't function correctly in the minimal chroot; just disable them.
unset NIX_BUILD_HOOK
# Make the build below copy paths from the CD if possible. Note that
# /tmp/root in the chroot is the root of the CD.
export NIX_OTHER_STORES=/tmp/root/nix:$NIX_OTHER_STORES
p=@nix@/libexec/nix/substituters
export NIX_SUBSTITUTERS=$p/copy-from-other-stores.pl:$p/download-from-binary-cache.pl
system_closure="$tmpdir/system.closure"
if [ -z "$closure" ]; then
# Get the absolute path to the NixOS/Nixpkgs sources.
nixpkgs="$(readlink -f $(nix-instantiate --find-file nixpkgs))"
nixEnvAction="-f <nixpkgs/nixos> --set -A system"
expr="(import <nixpkgs/nixos> {}).system"
system_root="$(nix-build -E "$expr")"
system_closure="$(closure "$expr")"
else
nixpkgs=""
nixEnvAction="--set $closure"
system_root=$closure
# Create a temporary file ending in .closure (so nixos-prepare-root knows to --import it) to transport the store closure
# to the filesytem we're preparing. Also delete it on exit!
nix-store --export $(nix-store -qR $closure) > $system_closure
fi
# Build the specified Nix expression in the target store and install
# it into the system configuration profile.
echo "building the system configuration..."
NIX_PATH="nixpkgs=/tmp/root/$nixpkgs:nixos-config=$NIXOS_CONFIG" NIXOS_CONFIG= \
chroot $mountPoint @nix@/bin/nix-env \
"${extraBuildFlags[@]}" -p /nix/var/nix/profiles/system $nixEnvAction
channel_root="$(nix-env -p /nix/var/nix/profiles/per-user/root/channels -q nixos --no-name --out-path 2>/dev/null || echo -n "")"
channel_closure="$tmpdir/channel.closure"
nix-store --export $channel_root > $channel_closure
# Populate the target root directory with the basics
@prepare_root@/bin/nixos-prepare-root $mountPoint $channel_root $system_root @nixClosure@ $system_closure $channel_closure
# Copy the NixOS/Nixpkgs sources to the target as the initial contents
# of the NixOS channel.
mkdir -m 0755 -p $mountPoint/nix/var/nix/profiles
mkdir -m 1777 -p $mountPoint/nix/var/nix/profiles/per-user
mkdir -m 0755 -p $mountPoint/nix/var/nix/profiles/per-user/root
srcs=$(nix-env "${extraBuildFlags[@]}" -p /nix/var/nix/profiles/per-user/root/channels -q nixos --no-name --out-path 2>/dev/null || echo -n "")
if [ -z "$noChannelCopy" ] && [ -n "$srcs" ]; then
echo "copying NixOS/Nixpkgs sources..."
chroot $mountPoint @nix@/bin/nix-env \
"${extraBuildFlags[@]}" -p /nix/var/nix/profiles/per-user/root/channels -i "$srcs" --quiet
fi
mkdir -m 0700 -p $mountPoint/root/.nix-defexpr
ln -sfn /nix/var/nix/profiles/per-user/root/channels $mountPoint/root/.nix-defexpr/channels
# Get rid of the /etc bind mounts.
for f in /etc/passwd /etc/group; do [ -f "$f" ] && umount $mountPoint/$f; done
# nixos-prepare-root doesn't currently do anything with file ownership, so we set it up here instead
chown @root_uid@:@nixbld_gid@ $mountPoint/nix/store
mount --rbind /dev $mountPoint/dev
mount --rbind /proc $mountPoint/proc
mount --rbind /sys $mountPoint/sys
# Grub needs an mtab.
ln -sfn /proc/mounts $mountPoint/etc/mtab
# Mark the target as a NixOS installation, otherwise
# switch-to-configuration will chicken out.
touch $mountPoint/etc/NIXOS
# Switch to the new system configuration. This will install Grub with
# a menu default pointing at the kernel/initrd/etc of the new
# configuration.

View File

@ -0,0 +1,105 @@
#! @shell@
# This script's goal is to perform all "static" setup of a filesystem structure from pre-built store paths. Everything
# in here should run in a non-root context and inside a Nix builder. It's designed primarily to be called from image-
# building scripts and from nixos-install, but because it makes very few assumptions about the context in which it runs,
# it could be useful in other contexts as well.
#
# Current behavior:
# - set up basic filesystem structure
# - make Nix store etc.
# - copy Nix, system, channel, and misceallaneous closures to target Nix store
# - register validity of all paths in the target store
# - set up channel and system profiles
# Ensure a consistent umask.
umask 0022
set -e
mountPoint="$1"
channel="$2"
system="$3"
shift 3
closures="$@"
PATH="@coreutils@/bin:@nix@/bin:@perl@/bin:@utillinux@/bin:@rsync@/bin"
if ! test -e "$mountPoint"; then
echo "mount point $mountPoint doesn't exist"
exit 1
fi
# Create a few of the standard directories in the target root directory.
mkdir -m 0755 -p $mountPoint/dev $mountPoint/proc $mountPoint/sys $mountPoint/etc $mountPoint/run $mountPoint/home
mkdir -m 01777 -p $mountPoint/tmp
mkdir -m 0755 -p $mountPoint/tmp/root
mkdir -m 0755 -p $mountPoint/var
mkdir -m 0700 -p $mountPoint/root
ln -s /run $mountPoint/var/run
# Create the necessary Nix directories on the target device
mkdir -m 0755 -p \
$mountPoint/nix/var/nix/gcroots \
$mountPoint/nix/var/nix/temproots \
$mountPoint/nix/var/nix/userpool \
$mountPoint/nix/var/nix/profiles \
$mountPoint/nix/var/nix/db \
$mountPoint/nix/var/log/nix/drvs
mkdir -m 1775 -p $mountPoint/nix/store
# All Nix operations below should operate on our target store, not /nix/store.
# N.B: this relies on Nix 1.12 or higher
export NIX_REMOTE=local?root=$mountPoint
# Copy our closures to the Nix store on the target mount point, unless they're already there.
for i in $closures; do
# We support closures both in the format produced by `nix-store --export` and by `exportReferencesGraph`,
# mostly because there doesn't seem to be a single format that can be produced outside of a nix build and
# inside one. See https://github.com/NixOS/nix/issues/1242 for more discussion.
if [[ "$i" =~ \.closure$ ]]; then
echo "importing serialized closure $i to $mountPoint..."
nix-store --import < $i
else
# There has to be a better way to do this, right?
echo "copying closure $i to $mountPoint..."
for j in $(perl @pathsFromGraph@ $i); do
echo " $j... "
rsync -a $j $mountPoint/nix/store/
done
nix-store --register-validity < $i
fi
done
# Create the required /bin/sh symlink; otherwise lots of things
# (notably the system() function) won't work.
if [ ! -x $mountPoint/@shell@ ]; then
echo "Error: @shell@ wasn't included in the closure" >&2
exit 1
fi
mkdir -m 0755 -p $mountPoint/bin
ln -sf @shell@ $mountPoint/bin/sh
echo "setting the system closure to '$system'..."
nix-env "${extraBuildFlags[@]}" -p $mountPoint/nix/var/nix/profiles/system --set "$system"
ln -sfn /nix/var/nix/profiles/system $mountPoint/run/current-system
# Copy the NixOS/Nixpkgs sources to the target as the initial contents of the NixOS channel.
mkdir -m 0755 -p $mountPoint/nix/var/nix/profiles
mkdir -m 1777 -p $mountPoint/nix/var/nix/profiles/per-user
mkdir -m 0755 -p $mountPoint/nix/var/nix/profiles/per-user/root
if [ -z "$noChannelCopy" ] && [ -n "$channel" ]; then
echo "copying channel..."
nix-env --option build-use-substitutes false "${extraBuildFlags[@]}" -p $mountPoint/nix/var/nix/profiles/per-user/root/channels --set "$channel" --quiet
fi
mkdir -m 0700 -p $mountPoint/root/.nix-defexpr
ln -sfn /nix/var/nix/profiles/per-user/root/channels $mountPoint/root/.nix-defexpr/channels
# Mark the target as a NixOS installation, otherwise switch-to-configuration will chicken out.
touch $mountPoint/etc/NIXOS

View File

@ -4,7 +4,6 @@
{ config, pkgs, modulesPath, ... }:
let
cfg = config.installer;
makeProg = args: pkgs.substituteAll (args // {
@ -17,6 +16,14 @@ let
src = ./nixos-build-vms/nixos-build-vms.sh;
};
nixos-prepare-root = makeProg {
name = "nixos-prepare-root";
src = ./nixos-prepare-root.sh;
nix = pkgs.nixUnstable;
inherit (pkgs) perl pathsFromGraph rsync utillinux coreutils;
};
nixos-install = makeProg {
name = "nixos-install";
src = ./nixos-install.sh;
@ -26,6 +33,7 @@ let
cacert = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
root_uid = config.ids.uids.root;
nixbld_gid = config.ids.gids.nixbld;
prepare_root = nixos-prepare-root;
nixClosure = pkgs.runCommand "closure"
{ exportReferencesGraph = ["refs" config.nix.package.out]; }
@ -69,6 +77,7 @@ in
environment.systemPackages =
[ nixos-build-vms
nixos-prepare-root
nixos-install
nixos-rebuild
nixos-generate-config
@ -77,7 +86,7 @@ in
];
system.build = {
inherit nixos-install nixos-generate-config nixos-option nixos-rebuild;
inherit nixos-install nixos-prepare-root nixos-generate-config nixos-option nixos-rebuild;
};
};

View File

@ -34,6 +34,12 @@ let
boot.loader.systemd-boot.enable = true;
''}
users.extraUsers.alice = {
isNormalUser = true;
home = "/home/alice";
description = "Alice Foobar";
};
hardware.enableAllFirmware = lib.mkForce false;
${replaceChars ["\n"] ["\n "] extraConfig}
@ -96,7 +102,7 @@ let
$machine->shutdown;
# Now see if we can boot the installation.
$machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}" });
$machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}", name => "boot-after-install" });
# For example to enter LUKS passphrase.
${preBootCommands}
@ -118,11 +124,17 @@ let
$machine->waitForUnit("swap.target");
$machine->succeed("cat /proc/swaps | grep -q /dev");
# Check that the store is in good shape
$machine->succeed("nix-store --verify --check-contents >&2");
# Check whether the channel works.
$machine->succeed("nix-env -iA nixos.procps >&2");
$machine->succeed("type -tP ps | tee /dev/stderr") =~ /.nix-profile/
or die "nix-env failed";
# Check that the daemon works, and that non-root users can run builds (this will build a new profile generation through the daemon)
$machine->succeed("su alice -l -c 'nix-env -iA nixos.procps' >&2");
# We need to a writable nix-store on next boot.
$machine->copyFileFromHost(
"${ makeConfig { inherit bootLoader grubVersion grubDevice grubIdentifier extraConfig; forceGrubReinstallCount = 1; } }",
@ -139,7 +151,7 @@ let
$machine->shutdown;
# Check whether a writable store build works
$machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}" });
$machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}", name => "rebuild-switch" });
${preBootCommands}
$machine->waitForUnit("multi-user.target");
$machine->copyFileFromHost(
@ -150,7 +162,7 @@ let
# And just to be sure, check that the machine still boots after
# "nixos-rebuild switch".
$machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}" });
$machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}", "boot-after-rebuild-switch" });
${preBootCommands}
$machine->waitForUnit("network.target");
$machine->shutdown;

View File

@ -131,12 +131,13 @@ in rec {
sha256 = "69e0f398affec2a14c47b46fec712906429c85312d5483be43e4c34da4f63f67";
};
# 1.11.8 doesn't yet have the patch to work on LLVM 4, so we patch it for now. Take this out once
# we move to a higher version. I'd pull the specific patch from upstream but it doesn't apply cleanly.
# Until 1.11.9 is released, we do this :)
patchPhase = ''
substituteInPlace src/libexpr/json-to-value.cc \
--replace 'std::less<Symbol>, gc_allocator<Value *>' \
'std::less<Symbol>, gc_allocator<std::pair<const Symbol, Value *> >'
sed -i '/if (settings.readOnlyMode) {/a curSchema = getSchema();' src/libstore/local-store.cc
'';
}) // { perl-bindings = nixStable; };