xen_4_8: init at 4.8.1

This commit adds the xen_4_8 package to be used instead of
xen (currently at 4.5.5):
 * Add packages xen_4_8, xen_4_8-slim and xen_4_8-light
 * Add packages qemu_xen_4_8 and qemu_xen_4_8-light to be used
   with xen_4_8-slim and xen_4_8-light respectively.
 * Add systemd to buildInputs of xen (it is required by oxenstored)
 * Adapt xen service to work with the new version of xen
 * Use xen-init-dom0 to initlilise dom0 in xen-store
 * Currently, the virtualisation.xen.stored option is ignored
   if xen 4.8 is used

nixos/modules/virtualisation/xen-dom0.nix

@@ -267,26 +267,36 @@ in
         mkdir -p /var/lib/xen # so we create them here unconditionally.
         grep -q control_d /proc/xen/capabilities
         '';
-      serviceConfig.ExecStart = ''
-        ${cfg.stored}${optionalString cfg.trace " -T /var/log/xen/xenstored-trace.log"} --no-fork
-        '';
+      serviceConfig = if cfg.package.version < "4.8" then
+        { ExecStart = ''
+            ${cfg.stored}${optionalString cfg.trace " -T /var/log/xen/xenstored-trace.log"} --no-fork
+            '';
+        } else {
+          ExecStart = ''
+            ${cfg.package}/etc/xen/scripts/launch-xenstore
+            '';
+          Type            = "notify";
+          RemainAfterExit = true;
+          NotifyAccess    = "all";
+        };
       postStart = ''
-        time=0
-        timeout=30
-        # Wait for xenstored to actually come up, timing out after 30 seconds
-        while [ $time -lt $timeout ] && ! `${cfg.package}/bin/xenstore-read -s / >/dev/null 2>&1` ; do
-            time=$(($time+1))
-            sleep 1
-        done
+        ${optionalString (cfg.package.version < "4.8") ''
+          time=0
+          timeout=30
+          # Wait for xenstored to actually come up, timing out after 30 seconds
+          while [ $time -lt $timeout ] && ! `${cfg.package}/bin/xenstore-read -s / >/dev/null 2>&1` ; do
+              time=$(($time+1))
+              sleep 1
+          done
 
-        # Exit if we timed out
-        if ! [ $time -lt $timeout ] ; then
-            echo "Could not start Xenstore Daemon"
-            exit 1
-        fi
-
-        ${cfg.package}/bin/xenstore-write "/local/domain/0/name" "Domain-0"
-        ${cfg.package}/bin/xenstore-write "/local/domain/0/domid" 0
+          # Exit if we timed out
+          if ! [ $time -lt $timeout ] ; then
+              echo "Could not start Xenstore Daemon"
+              exit 1
+          fi
+        ''}
+        echo "executing xen-init-dom0"
+        ${cfg.package}/lib/xen/bin/xen-init-dom0
         '';
     };
 
@@ -306,6 +316,7 @@ in
       description = "Xen Console Daemon";
       wantedBy = [ "multi-user.target" ];
       after = [ "xen-store.service" ];
+      requires = [ "xen-store.service" ];
       preStart = ''
         mkdir -p /var/run/xen
         ${optionalString cfg.trace "mkdir -p /var/log/xen"}
@@ -313,7 +324,9 @@ in
         '';
       serviceConfig = {
         ExecStart = ''
-          ${cfg.package}/bin/xenconsoled${optionalString cfg.trace " --log=all --log-dir=/var/log/xen"}
+          ${cfg.package}/bin/xenconsoled\
+            ${optionalString ((cfg.package.version >= "4.8")) " -i"}\
+            ${optionalString cfg.trace " --log=all --log-dir=/var/log/xen"}
           '';
       };
     };
@@ -323,6 +336,7 @@ in
       description = "Xen Qemu Daemon";
       wantedBy = [ "multi-user.target" ];
       after = [ "xen-console.service" ];
+      requires = [ "xen-store.service" ];
       serviceConfig.ExecStart = ''
         ${cfg.qemu} -xen-attach -xen-domid 0 -name dom0 -M xenpv \
            -nographic -monitor /dev/null -serial /dev/null -parallel /dev/null
@@ -333,7 +347,7 @@ in
     systemd.services.xen-watchdog = {
       description = "Xen Watchdog Daemon";
       wantedBy = [ "multi-user.target" ];
-      after = [ "xen-qemu.service" ];
+      after = [ "xen-qemu.service" "xen-domains.service" ];
       serviceConfig.ExecStart = "${cfg.package}/bin/xenwatchdogd 30 15";
       serviceConfig.Type = "forking";
       serviceConfig.RestartSec = "1";
@@ -426,6 +440,7 @@ in
       description = "Xen domains - automatically starts, saves and restores Xen domains";
       wantedBy = [ "multi-user.target" ];
       after = [ "xen-bridge.service" "xen-qemu.service" ];
+      requires = [ "xen-bridge.service" "xen-qemu.service" ];
       ## To prevent a race between dhcpcd and xend's bridge setup script
       ## (which renames eth* to peth* and recreates eth* as a virtual
       ## device), start dhcpcd after xend.

pkgs/applications/virtualization/xen/4.8.nix

@@ -0,0 +1,165 @@
+{ stdenv, callPackage, fetchurl, fetchpatch, fetchgit
+, withInternalQemu ? true
+, withInternalTraditionalQemu ? true
+, withInternalSeabios ? true
+, withSeabios ? !withInternalSeabios, seabios ? null
+, withInternalOVMF ? false # FIXME: tricky to build
+, withOVMF ? false, OVMF
+, withLibHVM ? true
+
+# qemu
+, udev, pciutils, xorg, SDL, pixman, acl, glusterfs, spice_protocol, usbredir
+, alsaLib
+, ... } @ args:
+
+assert withInternalSeabios -> !withSeabios;
+assert withInternalOVMF -> !withOVMF;
+
+with stdenv.lib;
+
+# Patching XEN? Check the XSAs at
+# https://xenbits.xen.org/xsa/
+# and try applying all the ones we don't have yet.
+
+let
+  xsaPatch = { name , sha256 }: (fetchpatch {
+    url = "https://xenbits.xen.org/xsa/xsa${name}.patch";
+    inherit sha256;
+  });
+
+  qemuDeps = [
+    udev pciutils xorg.libX11 SDL pixman acl glusterfs spice_protocol usbredir
+    alsaLib
+  ];
+in
+
+callPackage (import ./generic.nix (rec {
+  version = "4.8.1";
+
+  src = fetchurl {
+    url = "http://bits.xensource.com/oss-xen/release/${version}/xen-${version}.tar.gz";
+    sha256 = "158kb1w61jmwxi3fc560s4269hhpxrin9xhm60ljj52njhxias8x";
+  };
+
+  # Sources needed to build tools and firmwares.
+  xenfiles = optionalAttrs withInternalQemu {
+    "qemu-xen" = {
+      src = fetchgit {
+        url = https://xenbits.xen.org/git-http/qemu-xen.git;
+        rev = "refs/tags/qemu-xen-${version}";
+        sha256 = "1v19pp86kcgwvsbkrdrn4rlaj02i4054avw8k70w1m0rnwgcsdbs";
+      };
+      buildInputs = qemuDeps;
+      patches = [
+        #(xsaPatch {
+        #  name = "197-4.5-qemuu";
+        #  sha256 = "09gp980qdlfpfmxy0nk7ncyaa024jnrpzx9gpq2kah21xygy5myx";
+        #})
+      ];
+      meta.description = "Xen's fork of upstream Qemu";
+    };
+  } // optionalAttrs withInternalTraditionalQemu {
+    "qemu-xen-traditional" = {
+      src = fetchgit {
+        url = https://xenbits.xen.org/git-http/qemu-xen-traditional.git;
+        rev = "refs/tags/xen-${version}";
+        sha256 = "0mryap5y53r09m7qc0b821f717ghwm654r8c3ik1w7adzxr0l5qk";
+      };
+      buildInputs = qemuDeps;
+      patches = [
+      ];
+      postPatch = ''
+        substituteInPlace xen-hooks.mak \
+          --replace /usr/include/pci ${pciutils}/include/pci
+      '';
+      meta.description = "Xen's fork of upstream Qemu that uses old device model";
+    };
+  } // optionalAttrs withInternalSeabios {
+    "firmware/seabios-dir-remote" = {
+      src = fetchgit {
+        url = https://xenbits.xen.org/git-http/seabios.git;
+        rev = "f0cdc36d2f2424f6b40438f7ee7cc502c0eff4df";
+        sha256 = "1wq5pjkjrfzqnq3wyr15mcn1l4c563m65gdyf8jm97kgb13pwwfm";
+      };
+      patches = [ ./0000-qemu-seabios-enable-ATA_DMA.patch ];
+      meta.description = "Xen's fork of Seabios";
+    };
+  } // optionalAttrs withInternalOVMF {
+    "firmware/ovmf-dir-remote" = {
+      src = fetchgit {
+        url = https://xenbits.xen.org/git-http/ovmf.git;
+        rev = "173bf5c847e3ca8b42c11796ce048d8e2e916ff8";
+        sha256 = "07zmdj90zjrzip74fvd4ss8n8njk6cim85s58mc6snxmqqv7gmcr";
+      };
+      meta.description = "Xen's fork of OVMF";
+    };
+  } // {
+    # TODO: patch Xen to make this optional?
+    "firmware/etherboot/ipxe.git" = {
+      src = fetchgit {
+        url = https://git.ipxe.org/ipxe.git;
+        rev = "356f6c1b64d7a97746d1816cef8ca22bdd8d0b5d";
+        sha256 = "15n400vm3id5r8y3k6lrp9ab2911a9vh9856f5gvphkazfnmns09";
+      };
+      meta.description = "Xen's fork of iPXE";
+    };
+  } // optionalAttrs withLibHVM {
+    "xen-libhvm-dir-remote" = {
+      src = fetchgit {
+        name = "xen-libhvm";
+        url = https://github.com/michalpalka/xen-libhvm;
+        rev = "83065d36b36d6d527c2a4e0f5aaf0a09ee83122c";
+        sha256 = "1jzv479wvgjkazprqdzcdjy199azmx2xl3pnxli39kc5mvjz3lzd";
+      };
+      buildPhase = ''
+        make
+        cd biospt
+        cc -Wall -g -D_LINUX -Wstrict-prototypes biospt.c -o biospt -I../libhvm -L../libhvm -lxenhvm
+      '';
+      installPhase = ''
+        make install
+        cp biospt/biospt $out/bin/
+      '';
+      meta = {
+        description = ''
+          Helper library for reading ACPI and SMBIOS firmware values
+          from the host system for use with the HVM guest firmware
+          pass-through feature in Xen'';
+        license = licenses.bsd2;
+      };
+    };
+  };
+
+  configureFlags = []
+    ++ optional (!withInternalQemu) "--with-system-qemu" # use qemu from PATH
+    ++ optional (withInternalTraditionalQemu) "--enable-qemu-traditional"
+    ++ optional (!withInternalTraditionalQemu) "--disable-qemu-traditional"
+
+    ++ optional (withSeabios) "--with-system-seabios=${seabios}"
+    ++ optional (!withInternalSeabios && !withSeabios) "--disable-seabios"
+
+    ++ optional (withOVMF) "--with-system-ovmf=${OVMF.fd}/FV/OVMF.fd"
+    ++ optional (withInternalOVMF) "--enable-ovmf";
+
+  patches =
+    [ (xsaPatch {
+        name = "213-4.8";
+        sha256 = "0ia3zr6r3bqy2h48fdy7p0iz423lniy3i0qkdvzgv5a8m80darr2";
+      })
+      (xsaPatch {
+        name = "214";
+        sha256 = "0qapzx63z0yl84phnpnglpkxp6b9sy1y7cilhwjhxyigpfnm2rrk";
+      })
+    ];
+
+  # Fix build on Glibc 2.24.
+  NIX_CFLAGS_COMPILE = "-Wno-error=deprecated-declarations";
+
+  postPatch = ''
+    # Avoid a glibc >= 2.25 deprecation warnings that get fatal via -Werror.
+    sed 1i'#include <sys/sysmacros.h>' \
+      -i tools/blktap2/control/tap-ctl-allocate.c \
+      -i tools/libxl/libxl_device.c
+  '';
+
+})) args

pkgs/applications/virtualization/xen/generic.nix

@@ -14,7 +14,7 @@ config:
 # Scripts
 , coreutils, gawk, gnused, gnugrep, diffutils, multipath-tools
 , iproute, inetutils, iptables, bridge-utils, openvswitch, nbd, drbd
-, lvm2, utillinux, procps
+, lvm2, utillinux, procps, systemd
 
 # Documentation
 # python2Packages.markdown
@@ -61,7 +61,7 @@ stdenv.mkDerivation (rec {
     libiconv libuuid ncurses openssl perl python2Packages.python xz yajl zlib
 
     # oxenstored
-    ocamlPackages.findlib ocamlPackages.ocaml
+    ocamlPackages.findlib ocamlPackages.ocaml systemd
 
     # Python fixes
     python2Packages.wrapPython
@@ -153,12 +153,19 @@ stdenv.mkDerivation (rec {
     substituteInPlace tools/xenstat/Makefile \
       --replace /usr/include/curses.h ${ncurses.dev}/include/curses.h
 
-    # TODO: use this as a template and support our own if-up scripts instead?
-    substituteInPlace tools/hotplug/Linux/xen-backend.rules.in \
-      --replace "@XEN_SCRIPT_DIR@" $out/etc/xen/scripts
+    ${optionalString (config.version >= "4.8") ''
+      substituteInPlace tools/hotplug/Linux/launch-xenstore.in \
+        --replace /bin/mkdir mkdir
+    ''}
 
-    # blktap is not provided by xen, but by xapi
-    sed -i '/blktap/d' tools/hotplug/Linux/xen-backend.rules.in
+    ${optionalString (config.version < "4.6") ''
+      # TODO: use this as a template and support our own if-up scripts instead?
+      substituteInPlace tools/hotplug/Linux/xen-backend.rules.in \
+        --replace "@XEN_SCRIPT_DIR@" $out/etc/xen/scripts
+
+      # blktap is not provided by xen, but by xapi
+      sed -i '/blktap/d' tools/hotplug/Linux/xen-backend.rules.in
+    ''}
 
     ${withTools "patches" (name: x: ''
       ${concatMapStringsSep "\n" (p: ''
@@ -191,7 +198,7 @@ stdenv.mkDerivation (rec {
   '';
 
   installPhase = ''
-    mkdir -p $out $out/share
+    mkdir -p $out $out/share $out/share/man
     cp -prvd dist/install/nix/store/*/* $out/
     cp -prvd dist/install/boot $out/boot
     cp -prvd dist/install/etc $out

pkgs/applications/virtualization/xen/packages.nix

@@ -57,6 +57,56 @@ rec {
     };
   };
 
+  xen_4_8-vanilla = callPackage ./4.8.nix {
+    # At the very least included seabios and etherboot need gcc49,
+    # so we have to build all of it with gcc49.
+    stdenv = overrideCC stdenv gcc49;
+
+    meta = {
+      description = "vanilla";
+      longDescription = ''
+        Vanilla version of Xen. Uses forks of Qemu and Seabios bundled
+        with Xen. This gives vanilla experince, but wastes space and
+        build time: typical NixOS setup that runs lots of VMs will
+        build three different versions of Qemu when using this (two
+        forks and upstream).
+      '';
+    };
+  };
+
+  xen_4_8-slim = xen_4_8-vanilla.override {
+    withInternalQemu = false;
+    withInternalTraditionalQemu = true;
+    withInternalSeabios = false;
+    withSeabios = true;
+
+    meta = {
+      description = "slim";
+      longDescription = ''
+        Slimmed-down version of Xen that reuses nixpkgs packages as
+        much as possible. Different parts may get out of sync, but
+        this builds faster and uses less space than vanilla. Use with
+        `qemu_xen` from nixpkgs.
+      '';
+    };
+  };
+
+  xen_4_8-light = xen_4_8-vanilla.override {
+    withInternalQemu = false;
+    withInternalTraditionalQemu = false;
+    withInternalSeabios = false;
+    withSeabios = true;
+
+    meta = {
+      description = "light";
+      longDescription = ''
+        Slimmed-down version of Xen without `qemu-traditional` (you
+        don't need it if you don't know what it is). Use with
+        `qemu_xen-light` from nixpkgs.
+      '';
+    };
+  };
+
   xen-vanilla = xen_4_5-vanilla;
   xen-slim = xen_4_5-slim;
   xen-light = xen_4_5-light;

pkgs/top-level/all-packages.nix

@@ -12352,6 +12352,8 @@ with pkgs;
   # Building with `xen` instead of `xen-slim` is possible, but makes no sense.
   qemu_xen = lowPrio (qemu.override { x86Only = true; xenSupport = true; xen = xen-slim; });
   qemu_xen-light = lowPrio (qemu.override { x86Only = true; xenSupport = true; xen = xen-light; });
+  qemu_xen_4_8 = lowPrio (qemu.override { x86Only = true; xenSupport = true; xen = xen_4_8-slim; });
+  qemu_xen_4_8-light = lowPrio (qemu.override { x86Only = true; xenSupport = true; xen = xen_4_8-light; });
 
   qemu_test = lowPrio (qemu.override { x86Only = true; nixosTestRunner = true; });
 
@@ -16631,6 +16633,9 @@ with pkgs;
   xen = xenPackages.xen_4_5-vanilla;
   xen-slim = xenPackages.xen_4_5-slim;
   xen-light = xenPackages.xen_4_5-light;
+  xen_4_8 = xenPackages.xen_4_8-vanilla;
+  xen_4_8-slim = xenPackages.xen_4_8-slim;
+  xen_4_8-light = xenPackages.xen_4_8-light;
 
   xkbset = callPackage ../tools/X11/xkbset { };