From 195c3788ed80b2b54297bd679516729d7eb6415a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Mar 2008 17:08:40 +0000 Subject: [PATCH] * Cleanup. svn path=/nixpkgs/trunk/; revision=11166 --- .../vm/rpm/fedora-postinstall.sh | 5 - .../vm/rpm/fill-disk-with-rpms.sh | 43 ---- pkgs/build-support/vm/rpm/fill-disk-worker.sh | 65 ------ .../vm/rpm/redhat-postinstall.sh | 4 - pkgs/build-support/vm/rpm/run-in-uml-boot.sh | 59 ----- pkgs/build-support/vm/rpm/run-in-uml.sh | 35 --- pkgs/build-support/vm/test.nix | 57 +++++ pkgs/build-support/vm/vm.nix | 204 +++++++++--------- 8 files changed, 161 insertions(+), 311 deletions(-) delete mode 100644 pkgs/build-support/vm/rpm/fedora-postinstall.sh delete mode 100644 pkgs/build-support/vm/rpm/fill-disk-with-rpms.sh delete mode 100644 pkgs/build-support/vm/rpm/fill-disk-worker.sh delete mode 100644 pkgs/build-support/vm/rpm/redhat-postinstall.sh delete mode 100644 pkgs/build-support/vm/rpm/run-in-uml-boot.sh delete mode 100644 pkgs/build-support/vm/rpm/run-in-uml.sh create mode 100644 pkgs/build-support/vm/test.nix diff --git a/pkgs/build-support/vm/rpm/fedora-postinstall.sh b/pkgs/build-support/vm/rpm/fedora-postinstall.sh deleted file mode 100644 index 1ff582c9d320..000000000000 --- a/pkgs/build-support/vm/rpm/fedora-postinstall.sh +++ /dev/null @@ -1,5 +0,0 @@ -# Add missing symlink to /lib/libssl.so.0.9.7a (or similar). Curl -# needs it. -echo "adding libssl symlink..." -chroot $root /bin/sh -c "/bin/ln -v -s /lib/libssl.so.* /lib/libssl.so.4" -chroot $root /bin/sh -c "/bin/ln -v -s /lib/libcrypto.so.* /lib/libcrypto.so.4" diff --git a/pkgs/build-support/vm/rpm/fill-disk-with-rpms.sh b/pkgs/build-support/vm/rpm/fill-disk-with-rpms.sh deleted file mode 100644 index f2c96dfe30d0..000000000000 --- a/pkgs/build-support/vm/rpm/fill-disk-with-rpms.sh +++ /dev/null @@ -1,43 +0,0 @@ -source $stdenv/setup - -size=$(($size * 1024 * 1024)) - -image=$out/image -ensureDir $out - -echo "building image \`$fullName'" -echo "$name" > $out/name -echo "$fullName" > $out/full-name - -echo "creating sparse file of $size bytes in $image..." -dd if=/dev/zero of=$image bs=1 seek=$((size - 1)) count=1 - -echo "creating e2fs file system in $image..." -$e2fsprogs/sbin/mke2fs -F $image - -echo "initialising file system through UML..." -echo $PATH > path -echo $rpm > rpm -echo $sysvinit > sysvinit -echo $rpms > rpms -echo $postInstall > post-install -ln -s $worker $NIX_BUILD_TOP/worker # work around a stupid bug (no dots allowed in kernel arguments?) - -# UML requires an existing $HOME. -mkdir dummy -export HOME=$(pwd)/dummy - -# Run UML. -touch log -tail -n +0 -f log & # show UML output as it appears -tailPid=$! - -linux ubd0=$image root=/dev/root rootflags=/ rootfstype=hostfs \ - init="$SHELL $NIX_BUILD_TOP/worker $NIX_BUILD_TOP" con=null || true - -echo image is $image - -sleep 1 # drain `tail', hacky -kill $! - -if ! test -e success; then exit 1; fi diff --git a/pkgs/build-support/vm/rpm/fill-disk-worker.sh b/pkgs/build-support/vm/rpm/fill-disk-worker.sh deleted file mode 100644 index 138a57b1c004..000000000000 --- a/pkgs/build-support/vm/rpm/fill-disk-worker.sh +++ /dev/null @@ -1,65 +0,0 @@ -set -e -set -x - -export NIX_BUILD_TOP=$1 - -export PATH=$(< $NIX_BUILD_TOP/path) -export sysvinit=$(< $NIX_BUILD_TOP/sysvinit) -export postInstall=$(< $NIX_BUILD_TOP/post-install) -export rpm=$(< $NIX_BUILD_TOP/rpm) - -cd $NIX_BUILD_TOP - -root=$NIX_BUILD_TOP/root - -echo "remounting / writeable..." -mount -n -o remount,rw dummy / - -exec > log 2>&1 - -echo "creating /dev..." -mount -n -t tmpfs none /dev -mknod /dev/ubd0 b 98 0 - -echo "mounting image on $root..." -mkdir root -mount -n -t ext2 /dev/ubd0 $root - -echo "making directories..." -mkdir --mode 0755 $root/dev -mkdir --mode 0755 $root/proc - -echo "initialising RPM DB..." -rpm="$rpm/bin/rpm --root $root --dbpath /var/lib/rpm" -$rpm --initdb - -echo "installing RPMs..." -$rpm --noscripts --notriggers --nodeps -ihv $(cat rpms) - -if test -n "$postInstall"; then - echo "running post-install script" - . $postInstall -fi - -# Get rid of the Berkeley DB environment so that older RPM versions -# (using older versions of BDB) will still work. -rm -f root/var/lib/rpm/__db.* - -# We need to make devices because modern distributions (e.g., Fedora -# Core 3) use udev to create devices dynamically. We run into *very* -# subtle bugs if /dev/null in particular doesn't exist ;-) -echo "making devices..." -test -e $root/dev/null || mknod -m 0666 $root/dev/null c 1 3 -test -e $root/dev/zero || mknod -m 0666 $root/dev/zero c 1 5 -test -e $root/dev/tty || mknod -m 0600 $root/dev/tty c 5 0 - -echo "unmounting..." -umount -n $root - -touch $NIX_BUILD_TOP/success - -echo "syncing..." -sync - -echo "halting..." -$sysvinit/sbin/halt -d -f diff --git a/pkgs/build-support/vm/rpm/redhat-postinstall.sh b/pkgs/build-support/vm/rpm/redhat-postinstall.sh deleted file mode 100644 index 49a3400ec243..000000000000 --- a/pkgs/build-support/vm/rpm/redhat-postinstall.sh +++ /dev/null @@ -1,4 +0,0 @@ -# This is needed by curl. -echo "adding /usr/kerberos/lib to ld.so.conf..." -echo "/usr/kerberos/lib" >> $root/etc/ld.so.conf -chroot $root /sbin/ldconfig diff --git a/pkgs/build-support/vm/rpm/run-in-uml-boot.sh b/pkgs/build-support/vm/rpm/run-in-uml-boot.sh deleted file mode 100644 index 46c15331305e..000000000000 --- a/pkgs/build-support/vm/rpm/run-in-uml-boot.sh +++ /dev/null @@ -1,59 +0,0 @@ -set -e -set -x - -# Restore the original variables. -export NIX_BUILD_TOP=$1 -. $NIX_BUILD_TOP/initial-env-vars - -export PATH=$(< $NIX_BUILD_TOP/path) - -export NIX_STORE=/nix/store # !!! assuming that Nix store = /nix/store - -cd $NIX_BUILD_TOP - -root=$NIX_BUILD_TOP/root - -echo "remounting / writeable..." -mount -n -o remount,rw dummy / - -exec > log 2>&1 - -echo "creating /dev..." -mount -n -t tmpfs none /dev -mknod /dev/ubd0 b 98 0 - -echo "mounting image on $root..." -mkdir root -mount -n -t ext2 /dev/ubd0 $root - -echo "switching roots" -mkdir $root/hostfs - -# Make the Nix store available in the UML file system as a symlink to /hostfs/$NIX_STORE. -mkdir -p $root/$(dirname $NIX_STORE) -ln -s /hostfs/$NIX_STORE $root/$NIX_STORE - -$utillinux/sbin/pivot_root $root $root/hostfs -cd / -# !!! we should also chroot here (according to the pivot_root manpage) - -echo "mounting /proc..." -mount -n -t proc none /proc - -#mount --bind /hostfs/tmp /tmp - -cd /tmp -export TMPDIR=/tmp -export TMP_DIR=/tmp -#cd $NIX_BUILD_TOP - -# Call the actual builder. -if sh -e $actualBuilder; then - touch /hostfs/$NIX_BUILD_TOP/success -fi - -echo "syncing..." -sync - -echo "halting..." -$sysvinit/sbin/halt -d -f diff --git a/pkgs/build-support/vm/rpm/run-in-uml.sh b/pkgs/build-support/vm/rpm/run-in-uml.sh deleted file mode 100644 index 3f1c7c0d6e81..000000000000 --- a/pkgs/build-support/vm/rpm/run-in-uml.sh +++ /dev/null @@ -1,35 +0,0 @@ -export > initial-env-vars - -source $stdenv/setup - -ensureDir $out/nix-support -echo "$system" > $out/nix-support/system -cat $image/name > $out/nix-support/name -cat $image/full-name > $out/nix-support/full-name - -echo $PATH > path -ln -s $boot $NIX_BUILD_TOP/worker # work around a stupid bug (no dots allowed in kernel arguments?) - -# UML requires an existing $HOME. -mkdir dummy -export HOME=$(pwd)/dummy - -# Run UML. -startLog "uml-run" -touch log -tail -n +0 -f log & # show UML output as it appears -tailPid=$! - -echo "running UML using image $image/image ($(cat $image/full-name))" -linux mem=384M ubd0=cow,$image/image root=/dev/root rootflags=/ rootfstype=hostfs \ - init="$SHELL $NIX_BUILD_TOP/worker $NIX_BUILD_TOP" con=null || true -echo "UML finished" - -sleep 1 # drain `tail', hacky -kill $! -stopLog - -if ! test -e success; then - echo "UML build script failed" - exit 1 -fi diff --git a/pkgs/build-support/vm/test.nix b/pkgs/build-support/vm/test.nix new file mode 100644 index 000000000000..e25704612dcc --- /dev/null +++ b/pkgs/build-support/vm/test.nix @@ -0,0 +1,57 @@ +with import ../../.. {}; +with import ./vm.nix; + +rec { + + + # Run the PatchELF derivation in a VM. + buildPatchelfInVM = runInLinuxVM patchelf; + + + rpmImage = fillDiskWithRPMs { + name = "fedora-image"; + fullName = "Fedora Core 3"; + size = 1024; + rpms = import ./rpm/fedora-3-packages.nix {inherit fetchurl;}; + }; + + + testRPMImage = makeImageTestScript rpmImage; + + + buildPatchelfRPM = buildRPM { + name = "patchelf-rpm"; + src = patchelf.src; + diskImage = rpmImage; + }; + + + ubuntuImage = fillDiskWithDebs { + name = "ubuntu-image"; + fullName = "Ubuntu 7.10 Gutsy"; + size = 256; + debs = import ./deb/ubuntu-7.10-gutsy-i386.nix {inherit fetchurl;}; + }; + + + debianImage = fillDiskWithDebs { + name = "debian-image"; + fullName = "Debian 4.0r3 Etch"; + size = 256; + debs = import ./deb/debian-4.0r3-etch-i386.nix {inherit fetchurl;}; + }; + + + testUbuntuImage = makeImageTestScript ubuntuImage; + + + buildInDebian = runInLinuxImage (stdenv.mkDerivation { + name = "deb-compile"; + src = nixUnstable.src; + diskImage = debianImage; + memSize = 512; + }); + + + +} \ No newline at end of file diff --git a/pkgs/build-support/vm/vm.nix b/pkgs/build-support/vm/vm.nix index cc53b0c84179..13b9ed5ea756 100644 --- a/pkgs/build-support/vm/vm.nix +++ b/pkgs/build-support/vm/vm.nix @@ -176,7 +176,7 @@ rec { PATH=${coreutils}/bin:${kvm}/bin - diskImage=/dev/null + diskImage=''${diskImage:-/dev/null} eval "$preVM" @@ -201,8 +201,52 @@ rec { exit $(cat in-vm-exit) ''; - - # Modify the given derivation to perform it in a virtual machine. + + createEmptyImage = {size, fullName}: '' + mkdir $out + diskImage=$out/image + qemu-img create -f qcow $diskImage "${toString size}M" + + mkdir $out/nix-support + echo ${fullName} > $out/nix-support/full-name + ''; + + + createRootFS = '' + mkdir /mnt + ${e2fsprogs}/sbin/mke2fs -F /dev/sda + ${klibcShrunk}/bin/mount -t ext2 /dev/sda /mnt + + if test -e /mnt/.debug; then + exec ${bash}/bin/sh + fi + touch /mnt/.debug + + mkdir /mnt/proc /mnt/dev /mnt/sys /mnt/bin + ''; + + + /* Run a derivation in a Linux virtual machine (using Qemu/KVM). By + default, there is no disk image; the root filesystem is a tmpfs, + and /nix/store is shared with the host (via the CIFS protocol to + a Samba instance automatically started by Qemu). Thus, any pure + Nix derivation should run unmodified, e.g. the call + + runInLinuxVM patchelf + + will build the derivation `patchelf' inside a VM. The attribute + `preVM' can optionally contain a shell command to be evaluated + *before* the VM is started (i.e., on the host). The attribute + `memSize' specifies the memory size of the VM in megabytes, + defaulting to 256. The attribute `diskImage' can optionally + specify a file system image to be attached to /dev/sda. (Note + that currently we expect the image to contain a filesystem, not a + full disk image with a partition table etc.) + + If the build fails and Nix is run with the `-K' option, a script + `run-vm' will be left behind in the temporary build directory + that allows you to boot into the VM and debug it interactively. */ + runInLinuxVM = attrs: derivation (removeAttrs attrs ["meta" "passthru" "outPath" "drvPath"] // { builder = "${bash}/bin/sh"; args = ["-e" vmRunCommand]; @@ -211,28 +255,50 @@ rec { QEMU_OPTS = "-m ${toString (if attrs ? memSize then attrs.memSize else 256)}"; }); - - test = runInLinuxVM patchelf; + + /* Like runInLinuxVM, but run the build not using the stdenv from + the Nix store, but using the tools provided by /bin, /usr/bin + etc. from the specified filesystem image, which typically is a + filesystem containing a non-NixOS Linux distribution. */ + + runInLinuxImage = attrs: runInLinuxVM (attrs // { + mountDisk = true; + + /* Mount `image' as the root FS, but use a temporary copy-on-write + image since we don't want to (and can't) write to `image'. */ + preVM = '' + diskImage=$(pwd)/image + origImage=${attrs.diskImage} + if test -d "$origImage"; then origImage="$origImage/image"; fi + qemu-img create -b "$origImage" -f qcow $diskImage + ''; + + /* Inside the VM, run the stdenv setup script normally, but at the + very end set $PATH and $SHELL to the `native' paths for the + distribution inside the VM. */ + postHook = '' + PATH=/usr/bin:/bin:/usr/sbin:/sbin + SHELL=/bin/sh + ''; + + /* Don't run Nix-specific build steps like patchelf. */ + fixupPhase = "true"; + }); + /* Create a filesystem image of the specified size and fill it with + a set of RPM packages. */ + fillDiskWithRPMs = {size ? 1024, rpms, name, fullName, postInstall ? null}: runInLinuxVM (stdenv.mkDerivation { inherit name postInstall rpms; - preVM = '' - mkdir $out - diskImage=$out/image - qemu-img create -f qcow $diskImage "${toString size}M" - ''; + preVM = createEmptyImage {inherit size fullName;}; buildCommand = '' - mkdir /mnt - ${e2fsprogs}/sbin/mke2fs -F /dev/sda - ${klibcShrunk}/bin/mount -t ext2 /dev/sda /mnt - - mkdir /mnt/proc /mnt/dev /mnt/sys + ${createRootFS} echo "initialising RPM DB..." rpm="${rpm}/bin/rpm --root /mnt --dbpath /var/lib/rpm" @@ -254,21 +320,16 @@ rec { echo "running post-install script..." eval "$postInstall" + rm /mnt/.debug + ${klibcShrunk}/bin/umount /mnt ''; }); - test2 = fillDiskWithRPMs { - size = 1024; - name = "testY"; - fullName = "Test Image"; - rpms = import ./rpm/fedora-3-packages.nix {inherit fetchurl;}; - }; - - - # Generates a script that can be used to run an interactive session - # in the given image. + /* Generate a script that can be used to run an interactive session + in the given image. */ + makeImageTestScript = image: writeScript "image-test" '' #! ${bash}/bin/sh if test -z "$1"; then @@ -287,39 +348,36 @@ rec { ${qemuCommand} ''; - test3 = makeImageTestScript test2; - - buildRPM = runInLinuxVM (stdenv.mkDerivation { - name = "rpm-test"; + /* Build RPM packages from the tarball `src' in the Linux + distribution installed in the filesystem `diskImage'. The + tarball must contain an RPM specfile. */ - mountDisk = true; + buildRPM = attrs: runInLinuxImage (stdenv.mkDerivation (attrs // { + phases = "buildPhase installPhase"; - preVM = '' - diskImage=$(pwd)/image - qemu-img create -b ${test2}/image -f qcow $diskImage - ''; - - src = patchelf.src; - - buildCommand = '' - PATH=/usr/bin:/bin:/usr/sbin:/sbin - - echo ${patchelf.src} - + buildPhase = '' + # Hacky: RPM looks for .spec inside the tarball, so + # strip off the hash. stripHash "$src" srcName="$strippedName" ln -s "$src" "$srcName" rpmbuild -vv -ta "$srcName" + ''; + installPhase = '' ensureDir $out/rpms find /usr/src -name "*.rpm" -exec cp {} $out/rpms \; ''; - }); + })); - # !!! should probably merge this with fillDiskWithRPMs. + /* Create a filesystem image of the specified size and fill it with + a set of Debian packages. `debs' must be a list of list of + .deb files, namely, the Debian packages grouped together into + strongly connected components. See deb/deb-closure.nix. */ + fillDiskWithDebs = {size ? 1024, debs, name, fullName, postInstall ? null}: @@ -328,23 +386,10 @@ rec { debs = (lib.intersperse "|" debs); - preVM = '' - mkdir $out - diskImage=$out/image - qemu-img create -f qcow $diskImage "${toString size}M" - ''; + preVM = createEmptyImage {inherit size fullName;}; buildCommand = '' - mkdir /mnt - ${e2fsprogs}/sbin/mke2fs -F /dev/sda - ${klibcShrunk}/bin/mount -t ext2 /dev/sda /mnt - - if test -e /mnt/.debug; then - exec ${bash}/bin/sh - fi - touch /mnt/.debug - - mkdir /mnt/proc /mnt/dev /mnt/sys /mnt/bin + ${createRootFS} echo "initialising Debian DB..." PATH=$PATH:${dpkg}/bin:${dpkg}/sbin:${glibc}/sbin @@ -406,45 +451,4 @@ rec { }); - test4 = fillDiskWithDebs { - size = 256; - name = "deb-test"; - fullName = "Ubuntu Test Image"; - debs = import ./deb/ubuntu-7.10-gutsy-i386.nix {inherit fetchurl;}; - }; - - test5 = makeImageTestScript test4; - - - test6 = runInLinuxVM (stdenv.mkDerivation { - name = "deb-compile"; - - mountDisk = true; - - preVM = '' - diskImage=$(pwd)/image - qemu-img create -b ${test7}/image -f qcow $diskImage - ''; - - src = nixUnstable.src; - - postHook = '' - PATH=/usr/bin:/bin:/usr/sbin:/sbin - SHELL=/bin/sh - ''; - - fixupPhase = "true"; - - memSize = 512; - }); - - - test7 = fillDiskWithDebs { - size = 256; - name = "deb-test"; - fullName = "Debian Test Image"; - debs = import ./deb/debian-4.0r3-etch-i386.nix {inherit fetchurl;}; - }; - - }