From b3dfff282b114daf304aa4c97a2532cd9b6b8ca3 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Thu, 3 Feb 2022 09:56:55 -0500
Subject: [PATCH 01/16] openstack-image: minor cleanups

Format and remove unnecessary `with lib;`.
---
 .../maintainers/scripts/openstack/openstack-image.nix | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/nixos/maintainers/scripts/openstack/openstack-image.nix b/nixos/maintainers/scripts/openstack/openstack-image.nix
index 3255e7f3d44d..6638eb099938 100644
--- a/nixos/maintainers/scripts/openstack/openstack-image.nix
+++ b/nixos/maintainers/scripts/openstack/openstack-image.nix
@@ -1,14 +1,11 @@
 # nix-build '<nixpkgs/nixos>' -A config.system.build.openstackImage --arg configuration "{ imports = [ ./nixos/maintainers/scripts/openstack/openstack-image.nix ]; }"
 
 { config, lib, pkgs, ... }:
-
-with lib;
-
 {
-  imports =
-    [ ../../../modules/installer/cd-dvd/channel.nix
-      ../../../modules/virtualisation/openstack-config.nix
-    ];
+  imports = [
+    ../../../modules/installer/cd-dvd/channel.nix
+    ../../../modules/virtualisation/openstack-config.nix
+  ];
 
   system.build.openstackImage = import ../../../lib/make-disk-image.nix {
     inherit lib config;

From e9f015480c0573736cc87b5a90588a0b92a507d3 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Thu, 3 Feb 2022 09:57:01 -0500
Subject: [PATCH 02/16] openstack-image: make it easy to disable copying the
 channel to improve iteration time

---
 .../maintainers/scripts/openstack/openstack-image.nix  | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/nixos/maintainers/scripts/openstack/openstack-image.nix b/nixos/maintainers/scripts/openstack/openstack-image.nix
index 6638eb099938..6728a98758b8 100644
--- a/nixos/maintainers/scripts/openstack/openstack-image.nix
+++ b/nixos/maintainers/scripts/openstack/openstack-image.nix
@@ -1,14 +1,18 @@
 # nix-build '<nixpkgs/nixos>' -A config.system.build.openstackImage --arg configuration "{ imports = [ ./nixos/maintainers/scripts/openstack/openstack-image.nix ]; }"
 
 { config, lib, pkgs, ... }:
+let
+  copyChannel = true;
+in
 {
   imports = [
-    ../../../modules/installer/cd-dvd/channel.nix
     ../../../modules/virtualisation/openstack-config.nix
-  ];
+  ] ++ (lib.optional copyChannel ../../../modules/installer/cd-dvd/channel.nix);
+
+  documentation.enable = copyChannel;
 
   system.build.openstackImage = import ../../../lib/make-disk-image.nix {
-    inherit lib config;
+    inherit lib config copyChannel;
     additionalSpace = "1024M";
     pkgs = import ../../../.. { inherit (pkgs) system; }; # ensure we use the regular qemu-kvm package
     format = "qcow2";

From b8fe792394757f415c71c6cf20de10899c2388b1 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Thu, 3 Feb 2022 10:39:52 -0500
Subject: [PATCH 03/16] openstack-image-zfs: init

---
 .../scripts/openstack/openstack-image-zfs.nix | 115 ++++++++++++++++++
 1 file changed, 115 insertions(+)
 create mode 100644 nixos/maintainers/scripts/openstack/openstack-image-zfs.nix

diff --git a/nixos/maintainers/scripts/openstack/openstack-image-zfs.nix b/nixos/maintainers/scripts/openstack/openstack-image-zfs.nix
new file mode 100644
index 000000000000..dbd088e04141
--- /dev/null
+++ b/nixos/maintainers/scripts/openstack/openstack-image-zfs.nix
@@ -0,0 +1,115 @@
+# nix-build '<nixpkgs/nixos>' -A config.system.build.openstackImage --arg configuration "{ imports = [ ./nixos/maintainers/scripts/openstack/openstack-image.nix ]; }"
+
+{ config, lib, pkgs, ... }:
+let
+  inherit (lib) mkOption types;
+  copyChannel = false;
+  cfg = config.openstackImage;
+  imageBootMode = if config.openstack.efi then "uefi" else "legacy-bios";
+in
+{
+  imports = [
+    ../../../modules/virtualisation/openstack-config.nix
+  ] ++ (lib.optional copyChannel ../../../modules/installer/cd-dvd/channel.nix);
+
+
+  options.openstackImage = {
+    name = mkOption {
+      type = types.str;
+      description = "The name of the generated derivation";
+      default = "nixos-openstack-image-${config.system.nixos.label}-${pkgs.stdenv.hostPlatform.system}";
+    };
+
+    sizeMB = mkOption {
+      type = types.int;
+      default = 8192;
+      description = "The size in MB of the image";
+    };
+
+    format = mkOption {
+      type = types.enum [ "raw" "qcow2" "vpc" ];
+      default = "vpc";
+      description = "The image format to output";
+    };
+  };
+
+  config = {
+    documentation.enable = copyChannel;
+    openstack = {
+      efi = true;
+      zfs = {
+        enable = true;
+        datasets = {
+          "tank/system/root".mount = "/";
+          "tank/system/var".mount = "/var";
+          "tank/local/nix".mount = "/nix";
+          "tank/user/home".mount = "/home";
+        };
+      };
+    };
+
+    system.build.openstackImage' = import ../../../lib/make-disk-image.nix {
+      inherit lib config copyChannel;
+      additionalSpace = "1024M";
+      pkgs = import ../../../.. { inherit (pkgs) system; }; # ensure we use the regular qemu-kvm package
+      format = "qcow2";
+
+    };
+
+    system.build.openstackImage = import ../../../lib/make-zfs-image.nix {
+      inherit lib config;
+      inherit (cfg) contents format name;
+      pkgs = import ../../../.. { inherit (pkgs) system; }; # ensure we use the regular qemu-kvm package
+
+      configFile = pkgs.writeText "configuration.nix"
+        ''
+          {
+            imports = [ <nixpkgs/nixos/modules/virtualisation/openstack-config.nix> ];
+          }
+        '';
+
+      includeChannel = copyChannel;
+
+      bootSize = 1000;
+
+      rootSize = cfg.sizeMB;
+      rootPoolProperties = {
+        ashift = 12;
+        autoexpand = "on";
+      };
+
+      datasets = config.openstack.zfs.datasets;
+
+      postVM = ''
+         extension=''${rootDiskImage##*.}
+         friendlyName=$out/${cfg.name}
+         rootDisk="$friendlyName.root.$extension"
+         bootDisk="$friendlyName.boot.$extension"
+         mv "$rootDiskImage" "$rootDisk"
+         mv "$bootDiskImage" "$bootDisk"
+
+         mkdir -p $out/nix-support
+         echo "file ${cfg.format} $bootDisk" >> $out/nix-support/hydra-build-products
+         echo "file ${cfg.format} $rootDisk" >> $out/nix-support/hydra-build-products
+
+        ${pkgs.jq}/bin/jq -n \
+          --arg system_label ${lib.escapeShellArg config.system.nixos.label} \
+          --arg system ${lib.escapeShellArg pkgs.stdenv.hostPlatform.system} \
+          --arg root_logical_bytes "$(${pkgs.qemu}/bin/qemu-img info --output json "$rootDisk" | ${pkgs.jq}/bin/jq '."virtual-size"')" \
+          --arg boot_logical_bytes "$(${pkgs.qemu}/bin/qemu-img info --output json "$bootDisk" | ${pkgs.jq}/bin/jq '."virtual-size"')" \
+          --arg boot_mode "${imageBootMode}" \
+          --arg root "$rootDisk" \
+          --arg boot "$bootDisk" \
+         '{}
+           | .label = $system_label
+           | .boot_mode = $boot_mode
+           | .system = $system
+           | .disks.boot.logical_bytes = $boot_logical_bytes
+           | .disks.boot.file = $boot
+           | .disks.root.logical_bytes = $root_logical_bytes
+           | .disks.root.file = $root
+           ' > $out/nix-support/image-info.json
+      '';
+    };
+  };
+}

From 80b00ef02a8efc958e2a16a7264c147e2a8ffbf4 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Thu, 3 Feb 2022 11:03:38 -0500
Subject: [PATCH 04/16] openstack-options: init

---
 nixos/modules/module-list.nix                 |  1 +
 .../virtualisation/openstack-options.nix      | 71 +++++++++++++++++++
 2 files changed, 72 insertions(+)
 create mode 100644 nixos/modules/virtualisation/openstack-options.nix

diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix
index 1ad2592b1dd7..734d1dfa8dc3 100644
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@@ -1235,6 +1235,7 @@
   ./virtualisation/amazon-options.nix
   ./virtualisation/hyperv-guest.nix
   ./virtualisation/kvmgt.nix
+  ./virtualisation/openstack-options.nix
   ./virtualisation/openvswitch.nix
   ./virtualisation/parallels-guest.nix
   ./virtualisation/podman/default.nix
diff --git a/nixos/modules/virtualisation/openstack-options.nix b/nixos/modules/virtualisation/openstack-options.nix
new file mode 100644
index 000000000000..cbc779f27c8f
--- /dev/null
+++ b/nixos/modules/virtualisation/openstack-options.nix
@@ -0,0 +1,71 @@
+{ config, lib, pkgs, ... }:
+let
+  inherit (lib) literalExpression types;
+in
+{
+  options = {
+    openstack = {
+      zfs = {
+        enable = lib.mkOption {
+          default = false;
+          internal = true;
+          description = ''
+            Whether the OpenStack instance uses a ZFS root.
+          '';
+        };
+
+        datasets = lib.mkOption {
+          description = ''
+            Datasets to create under the `tank` and `boot` zpools.
+
+            **NOTE:** This option is used only at image creation time, and
+            does not attempt to declaratively create or manage datasets
+            on an existing system.
+          '';
+
+          default = { };
+
+          type = types.attrsOf (types.submodule {
+            options = {
+              mount = lib.mkOption {
+                description = "Where to mount this dataset.";
+                type = types.nullOr types.string;
+                default = null;
+              };
+
+              properties = lib.mkOption {
+                description = "Properties to set on this dataset.";
+                type = types.attrsOf types.string;
+                default = { };
+              };
+            };
+          });
+        };
+      };
+
+      efi = lib.mkOption {
+        default = pkgs.stdenv.hostPlatform.isAarch64;
+        defaultText = literalExpression "pkgs.stdenv.hostPlatform.isAarch64";
+        internal = true;
+        description = ''
+          Whether the instance is using EFI.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf config.openstack.zfs.enable {
+    networking.hostId = lib.mkDefault "00000000";
+
+    fileSystems =
+      let
+        mountable = lib.filterAttrs (_: value: ((value.mount or null) != null)) config.openstack.zfs.datasets;
+      in
+      lib.mapAttrs'
+        (dataset: opts: lib.nameValuePair opts.mount {
+          device = dataset;
+          fsType = "zfs";
+        })
+        mountable;
+  };
+}

From e5a7d077c1a1d6a91e40634e9fd4841a91b4bb1f Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Thu, 3 Feb 2022 11:05:52 -0500
Subject: [PATCH 05/16] openstack-config: support a ZFS root with /boot perhaps
 coming from an ESP

---
 .../virtualisation/openstack-config.nix       | 25 +++++++++++++------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/nixos/modules/virtualisation/openstack-config.nix b/nixos/modules/virtualisation/openstack-config.nix
index d01e0f23aba1..a5aa4b38eb66 100644
--- a/nixos/modules/virtualisation/openstack-config.nix
+++ b/nixos/modules/virtualisation/openstack-config.nix
@@ -1,8 +1,8 @@
-{ pkgs, lib, ... }:
-
-with lib;
+{ config, pkgs, lib, ... }:
 
 let
+  inherit (lib) mkIf mkDefault;
+  cfg = config.openstack;
   metadataFetcher = import ./openstack-metadata-fetcher.nix {
     targetRoot = "/";
     wgetExtraOptions = "--retry-connrefused";
@@ -18,16 +18,27 @@ in
   ];
 
   config = {
-    fileSystems."/" = {
+    fileSystems."/" = mkIf (!cfg.zfs.enable) {
       device = "/dev/disk/by-label/nixos";
       fsType = "ext4";
       autoResize = true;
     };
 
+    fileSystems."/boot" = mkIf (cfg.efi || cfg.zfs.enable) {
+      # The ZFS image uses a partition labeled ESP whether or not we're
+      # booting with EFI.
+      device = "/dev/disk/by-label/ESP";
+      fsType = "vfat";
+    };
+
     boot.growPartition = true;
     boot.kernelParams = [ "console=ttyS0" ];
-    boot.loader.grub.device = "/dev/vda";
-    boot.loader.timeout = 0;
+    boot.loader.grub.device = if (!cfg.efi) then "/dev/vda" else "nodev";
+    boot.loader.grub.efiSupport = cfg.efi;
+    boot.loader.grub.efiInstallAsRemovable = cfg.efi;
+
+    services.zfs.expandOnBoot = mkIf cfg.zfs.enable "all";
+    boot.zfs.devNodes = mkIf cfg.zfs.enable "/dev/";
 
     # Allow root logins
     services.openssh = {
@@ -43,7 +54,7 @@ in
       path = [ pkgs.wget ];
       description = "Fetch Metadata on startup";
       wantedBy = [ "multi-user.target" ];
-      before = [ "apply-ec2-data.service" "amazon-init.service"];
+      before = [ "apply-ec2-data.service" "amazon-init.service" ];
       wants = [ "network-online.target" ];
       after = [ "network-online.target" ];
       script = metadataFetcher;

From 14304bfe40d8fa90f6a371cd6445a3ec7d09e0e9 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Thu, 24 Mar 2022 08:58:54 -0700
Subject: [PATCH 06/16] openstack-config: setup serial access on ttyS0

---
 nixos/modules/virtualisation/openstack-config.nix | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/nixos/modules/virtualisation/openstack-config.nix b/nixos/modules/virtualisation/openstack-config.nix
index a5aa4b38eb66..0b392cca68e1 100644
--- a/nixos/modules/virtualisation/openstack-config.nix
+++ b/nixos/modules/virtualisation/openstack-config.nix
@@ -11,7 +11,11 @@ in
 {
   imports = [
     ../profiles/qemu-guest.nix
+
+    # Note: While we do use the headless profile, we also explicitly
+    # turn on the serial console on ttyS0 below.
     ../profiles/headless.nix
+
     # The Openstack Metadata service exposes data on an EC2 API also.
     ./ec2-data.nix
     ./amazon-init.nix
@@ -36,6 +40,12 @@ in
     boot.loader.grub.device = if (!cfg.efi) then "/dev/vda" else "nodev";
     boot.loader.grub.efiSupport = cfg.efi;
     boot.loader.grub.efiInstallAsRemovable = cfg.efi;
+    boot.loader.timeout = 1;
+    boot.loader.grub.extraConfig = ''
+      serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1
+      terminal_output console serial
+      terminal_input console serial
+    '';
 
     services.zfs.expandOnBoot = mkIf cfg.zfs.enable "all";
     boot.zfs.devNodes = mkIf cfg.zfs.enable "/dev/";
@@ -47,6 +57,11 @@ in
       passwordAuthentication = mkDefault false;
     };
 
+    users.users.root.initialPassword = "foobar";
+
+    # Enable the serial console on ttyS0
+    systemd.services."serial-getty@ttyS0".enable = true;
+
     # Force getting the hostname from Openstack metadata.
     networking.hostName = mkDefault "";
 

From a8f41adbb753f1a94cfd9086f09b177cf11a7a4b Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Wed, 16 Feb 2022 13:56:51 -0500
Subject: [PATCH 07/16] amazon-image: use make-multi-disk-zfs-image

---
 nixos/lib/{make-zfs-image.nix => make-multi-disk-zfs-image.nix} | 0
 nixos/maintainers/scripts/ec2/amazon-image.nix                  | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename nixos/lib/{make-zfs-image.nix => make-multi-disk-zfs-image.nix} (100%)

diff --git a/nixos/lib/make-zfs-image.nix b/nixos/lib/make-multi-disk-zfs-image.nix
similarity index 100%
rename from nixos/lib/make-zfs-image.nix
rename to nixos/lib/make-multi-disk-zfs-image.nix
diff --git a/nixos/maintainers/scripts/ec2/amazon-image.nix b/nixos/maintainers/scripts/ec2/amazon-image.nix
index 6358ec68f7cf..2d89db0a7f34 100644
--- a/nixos/maintainers/scripts/ec2/amazon-image.nix
+++ b/nixos/maintainers/scripts/ec2/amazon-image.nix
@@ -73,7 +73,7 @@ in {
         }
       '';
 
-    zfsBuilder = import ../../../lib/make-zfs-image.nix {
+    zfsBuilder = import ../../../lib/make-multi-disk-zfs-image.nix {
       inherit lib config configFile;
       inherit (cfg) contents format name;
       pkgs = import ../../../.. { inherit (pkgs) system; }; # ensure we use the regular qemu-kvm package

From d4c502a94a03a66da48870f30588dc7d601c50f8 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Mon, 21 Feb 2022 14:56:56 -0500
Subject: [PATCH 08/16] openstack-metadata-fetcher: don't fail if any specific
 wget's fail

---
 nixos/modules/virtualisation/openstack-metadata-fetcher.nix | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/nixos/modules/virtualisation/openstack-metadata-fetcher.nix b/nixos/modules/virtualisation/openstack-metadata-fetcher.nix
index 25104bb47667..d62428b47a44 100644
--- a/nixos/modules/virtualisation/openstack-metadata-fetcher.nix
+++ b/nixos/modules/virtualisation/openstack-metadata-fetcher.nix
@@ -14,9 +14,9 @@
     wget ${wgetExtraOptions} "$@"
   }
 
-  wget_imds -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path
+  wget_imds -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path || true
   # When no user-data is provided, the OpenStack metadata server doesn't expose the user-data route.
   (umask 077 && wget_imds -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data || rm -f "$metaDir/user-data")
-  wget_imds -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname
-  wget_imds -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key
+  wget_imds -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname || true
+  wget_imds -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key || true
 ''

From 1c0b76b5c447ff0879ae5e57fa8af0cb5639bdd8 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 8 Mar 2022 09:21:41 -0500
Subject: [PATCH 09/16] openstack-image-zfs: don't support vpc type, default to
 qcow2

---
 nixos/maintainers/scripts/openstack/openstack-image-zfs.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nixos/maintainers/scripts/openstack/openstack-image-zfs.nix b/nixos/maintainers/scripts/openstack/openstack-image-zfs.nix
index dbd088e04141..714b24a9e046 100644
--- a/nixos/maintainers/scripts/openstack/openstack-image-zfs.nix
+++ b/nixos/maintainers/scripts/openstack/openstack-image-zfs.nix
@@ -27,8 +27,8 @@ in
     };
 
     format = mkOption {
-      type = types.enum [ "raw" "qcow2" "vpc" ];
-      default = "vpc";
+      type = types.enum [ "raw" "qcow2" ];
+      default = "qcow2";
       description = "The image format to output";
     };
   };

From 0a086bf7bf88d617f0cb126f8119e9e57b77af45 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 8 Mar 2022 09:22:38 -0500
Subject: [PATCH 10/16] openstack-config: enable tty1

This appears to be the "default" console for openstack, but my evidence is experience on just one provider.
---
 nixos/modules/virtualisation/openstack-config.nix | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/nixos/modules/virtualisation/openstack-config.nix b/nixos/modules/virtualisation/openstack-config.nix
index 0b392cca68e1..c5c832fb8105 100644
--- a/nixos/modules/virtualisation/openstack-config.nix
+++ b/nixos/modules/virtualisation/openstack-config.nix
@@ -13,7 +13,10 @@ in
     ../profiles/qemu-guest.nix
 
     # Note: While we do use the headless profile, we also explicitly
-    # turn on the serial console on ttyS0 below.
+    # turn on the serial console on tty1 below.
+    # Note that I could not find any documentation indicating tty1 was
+    # the correct choice. I picked tty1 because that is what one
+    # particular host was using.
     ../profiles/headless.nix
 
     # The Openstack Metadata service exposes data on an EC2 API also.
@@ -36,13 +39,13 @@ in
     };
 
     boot.growPartition = true;
-    boot.kernelParams = [ "console=ttyS0" ];
+    boot.kernelParams = [ "console=tty1" ];
     boot.loader.grub.device = if (!cfg.efi) then "/dev/vda" else "nodev";
     boot.loader.grub.efiSupport = cfg.efi;
     boot.loader.grub.efiInstallAsRemovable = cfg.efi;
     boot.loader.timeout = 1;
     boot.loader.grub.extraConfig = ''
-      serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1
+      serial --unit=1 --speed=115200 --word=8 --parity=no --stop=1
       terminal_output console serial
       terminal_input console serial
     '';
@@ -59,8 +62,8 @@ in
 
     users.users.root.initialPassword = "foobar";
 
-    # Enable the serial console on ttyS0
-    systemd.services."serial-getty@ttyS0".enable = true;
+    # Enable the serial console on tty1
+    systemd.services."serial-getty@tty1".enable = true;
 
     # Force getting the hostname from Openstack metadata.
     networking.hostName = mkDefault "";

From 9e3dab7d2e2f7e08baa65dfec78e27343608bdbd Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 8 Mar 2022 09:24:33 -0500
Subject: [PATCH 11/16] openstack-image-zfs: build a single-image ZFS root

---
 .../scripts/openstack/openstack-image-zfs.nix   | 17 +----------------
 1 file changed, 1 insertion(+), 16 deletions(-)

diff --git a/nixos/maintainers/scripts/openstack/openstack-image-zfs.nix b/nixos/maintainers/scripts/openstack/openstack-image-zfs.nix
index 714b24a9e046..b18b3265283f 100644
--- a/nixos/maintainers/scripts/openstack/openstack-image-zfs.nix
+++ b/nixos/maintainers/scripts/openstack/openstack-image-zfs.nix
@@ -48,15 +48,7 @@ in
       };
     };
 
-    system.build.openstackImage' = import ../../../lib/make-disk-image.nix {
-      inherit lib config copyChannel;
-      additionalSpace = "1024M";
-      pkgs = import ../../../.. { inherit (pkgs) system; }; # ensure we use the regular qemu-kvm package
-      format = "qcow2";
-
-    };
-
-    system.build.openstackImage = import ../../../lib/make-zfs-image.nix {
+    system.build.openstackImage = import ../../../lib/make-single-disk-zfs-image.nix {
       inherit lib config;
       inherit (cfg) contents format name;
       pkgs = import ../../../.. { inherit (pkgs) system; }; # ensure we use the regular qemu-kvm package
@@ -84,28 +76,21 @@ in
          extension=''${rootDiskImage##*.}
          friendlyName=$out/${cfg.name}
          rootDisk="$friendlyName.root.$extension"
-         bootDisk="$friendlyName.boot.$extension"
          mv "$rootDiskImage" "$rootDisk"
-         mv "$bootDiskImage" "$bootDisk"
 
          mkdir -p $out/nix-support
-         echo "file ${cfg.format} $bootDisk" >> $out/nix-support/hydra-build-products
          echo "file ${cfg.format} $rootDisk" >> $out/nix-support/hydra-build-products
 
         ${pkgs.jq}/bin/jq -n \
           --arg system_label ${lib.escapeShellArg config.system.nixos.label} \
           --arg system ${lib.escapeShellArg pkgs.stdenv.hostPlatform.system} \
           --arg root_logical_bytes "$(${pkgs.qemu}/bin/qemu-img info --output json "$rootDisk" | ${pkgs.jq}/bin/jq '."virtual-size"')" \
-          --arg boot_logical_bytes "$(${pkgs.qemu}/bin/qemu-img info --output json "$bootDisk" | ${pkgs.jq}/bin/jq '."virtual-size"')" \
           --arg boot_mode "${imageBootMode}" \
           --arg root "$rootDisk" \
-          --arg boot "$bootDisk" \
          '{}
            | .label = $system_label
            | .boot_mode = $boot_mode
            | .system = $system
-           | .disks.boot.logical_bytes = $boot_logical_bytes
-           | .disks.boot.file = $boot
            | .disks.root.logical_bytes = $root_logical_bytes
            | .disks.root.file = $root
            ' > $out/nix-support/image-info.json

From b4c495aeffb610cffd08e0000817f02742365dec Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 8 Mar 2022 09:25:26 -0500
Subject: [PATCH 12/16] openstack-image-zfs: make the generated
 configuration.nix valid

Drops the wrong expandOnBoot option (defaults to all) and mark ZFS as enabled.
---
 nixos/maintainers/scripts/openstack/openstack-image-zfs.nix | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/nixos/maintainers/scripts/openstack/openstack-image-zfs.nix b/nixos/maintainers/scripts/openstack/openstack-image-zfs.nix
index b18b3265283f..515cc5cb38b3 100644
--- a/nixos/maintainers/scripts/openstack/openstack-image-zfs.nix
+++ b/nixos/maintainers/scripts/openstack/openstack-image-zfs.nix
@@ -55,8 +55,9 @@ in
 
       configFile = pkgs.writeText "configuration.nix"
         ''
-          {
-            imports = [ <nixpkgs/nixos/modules/virtualisation/openstack-config.nix> ];
+          { modulesPath, ... }: {
+            imports = [ "''${modulesPath}/virtualisation/openstack-config.nix" ];
+            openstack.zfs.enable = true;
           }
         '';
 

From 8a5bdce5665e24c1656aab2235ac9bd556316adf Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 8 Mar 2022 09:25:44 -0500
Subject: [PATCH 13/16] make-single-disk-zfs-image: init

---
 nixos/lib/make-single-disk-zfs-image.nix | 322 +++++++++++++++++++++++
 1 file changed, 322 insertions(+)
 create mode 100644 nixos/lib/make-single-disk-zfs-image.nix

diff --git a/nixos/lib/make-single-disk-zfs-image.nix b/nixos/lib/make-single-disk-zfs-image.nix
new file mode 100644
index 000000000000..9310febd9179
--- /dev/null
+++ b/nixos/lib/make-single-disk-zfs-image.nix
@@ -0,0 +1,322 @@
+# Note: This is a private API, internal to NixOS. Its interface is subject
+# to change without notice.
+#
+# The result of this builder is a single disk image, partitioned like this:
+#
+#  * partition #1: a very small, 1MiB partition to leave room for Grub.
+#
+#  * partition #2: boot, a partition formatted with FAT to be used for /boot.
+#      FAT is chosen to support EFI.
+#
+#  * partition #3: nixos, a partition dedicated to a zpool.
+#
+# This single-disk approach does not satisfy ZFS's requirements for autoexpand,
+# however automation can expand it anyway. For example, with
+# `services.zfs.expandOnBoot`.
+{ lib
+, pkgs
+, # The NixOS configuration to be installed onto the disk image.
+  config
+
+, # size of the FAT partition, in megabytes.
+  bootSize ? 1024
+
+, # The size of the root partition, in megabytes.
+  rootSize ? 2048
+
+, # The name of the ZFS pool
+  rootPoolName ? "tank"
+
+, # zpool properties
+  rootPoolProperties ? {
+    autoexpand = "on";
+  }
+, # pool-wide filesystem properties
+  rootPoolFilesystemProperties ? {
+    acltype = "posixacl";
+    atime = "off";
+    compression = "on";
+    mountpoint = "legacy";
+    xattr = "sa";
+  }
+
+, # datasets, with per-attribute options:
+  # mount: (optional) mount point in the VM
+  # properties: (optional) ZFS properties on the dataset, like filesystemProperties
+  # Notes:
+  # 1. datasets will be created from shorter to longer names as a simple topo-sort
+  # 2. you should define a root's dataset's mount for `/`
+  datasets ? { }
+
+, # The files and directories to be placed in the target file system.
+  # This is a list of attribute sets {source, target} where `source'
+  # is the file system object (regular file or directory) to be
+  # grafted in the file system at path `target'.
+  contents ? [ ]
+
+, # The initial NixOS configuration file to be copied to
+  # /etc/nixos/configuration.nix. This configuration will be embedded
+  # inside a configuration which includes the described ZFS fileSystems.
+  configFile ? null
+
+, # Shell code executed after the VM has finished.
+  postVM ? ""
+
+, name ? "nixos-disk-image"
+
+, # Disk image format, one of qcow2, qcow2-compressed, vdi, vpc, raw.
+  format ? "raw"
+
+, # Include a copy of Nixpkgs in the disk image
+  includeChannel ? true
+}:
+let
+  formatOpt = if format == "qcow2-compressed" then "qcow2" else format;
+
+  compress = lib.optionalString (format == "qcow2-compressed") "-c";
+
+  filenameSuffix = "." + {
+    qcow2 = "qcow2";
+    vdi = "vdi";
+    vpc = "vhd";
+    raw = "img";
+  }.${formatOpt} or formatOpt;
+  rootFilename = "nixos.root${filenameSuffix}";
+
+  # FIXME: merge with channel.nix / make-channel.nix.
+  channelSources =
+    let
+      nixpkgs = lib.cleanSource pkgs.path;
+    in
+    pkgs.runCommand "nixos-${config.system.nixos.version}" { } ''
+      mkdir -p $out
+      cp -prd ${nixpkgs.outPath} $out/nixos
+      chmod -R u+w $out/nixos
+      if [ ! -e $out/nixos/nixpkgs ]; then
+        ln -s . $out/nixos/nixpkgs
+      fi
+      rm -rf $out/nixos/.git
+      echo -n ${config.system.nixos.versionSuffix} > $out/nixos/.version-suffix
+    '';
+
+  closureInfo = pkgs.closureInfo {
+    rootPaths = [ config.system.build.toplevel ]
+      ++ (lib.optional includeChannel channelSources);
+  };
+
+  modulesTree = pkgs.aggregateModules
+    (with config.boot.kernelPackages; [ kernel zfs ]);
+
+  tools = lib.makeBinPath (
+    with pkgs; [
+      config.system.build.nixos-enter
+      config.system.build.nixos-install
+      dosfstools
+      e2fsprogs
+      gptfdisk
+      nix
+      parted
+      utillinux
+      zfs
+    ]
+  );
+
+  hasDefinedMount = disk: ((disk.mount or null) != null);
+
+  stringifyProperties = prefix: properties: lib.concatStringsSep " \\\n" (
+    lib.mapAttrsToList
+      (
+        property: value: "${prefix} ${lib.escapeShellArg property}=${lib.escapeShellArg value}"
+      )
+      properties
+  );
+
+  featuresToProperties = features:
+    lib.listToAttrs
+      (builtins.map
+        (feature: {
+          name = "feature@${feature}";
+          value = "enabled";
+        })
+        features);
+
+  createDatasets =
+    let
+      datasetlist = lib.mapAttrsToList lib.nameValuePair datasets;
+      sorted = lib.sort (left: right: (lib.stringLength left.name) < (lib.stringLength right.name)) datasetlist;
+      cmd = { name, value }:
+        let
+          properties = stringifyProperties "-o" (value.properties or { });
+        in
+        "zfs create -p ${properties} ${name}";
+    in
+    lib.concatMapStringsSep "\n" cmd sorted;
+
+  mountDatasets =
+    let
+      datasetlist = lib.mapAttrsToList lib.nameValuePair datasets;
+      mounts = lib.filter ({ value, ... }: hasDefinedMount value) datasetlist;
+      sorted = lib.sort (left: right: (lib.stringLength left.value.mount) < (lib.stringLength right.value.mount)) mounts;
+      cmd = { name, value }:
+        ''
+          mkdir -p /mnt${lib.escapeShellArg value.mount}
+          mount -t zfs ${name} /mnt${lib.escapeShellArg value.mount}
+        '';
+    in
+    lib.concatMapStringsSep "\n" cmd sorted;
+
+  unmountDatasets =
+    let
+      datasetlist = lib.mapAttrsToList lib.nameValuePair datasets;
+      mounts = lib.filter ({ value, ... }: hasDefinedMount value) datasetlist;
+      sorted = lib.sort (left: right: (lib.stringLength left.value.mount) > (lib.stringLength right.value.mount)) mounts;
+      cmd = { name, value }:
+        ''
+          umount /mnt${lib.escapeShellArg value.mount}
+        '';
+    in
+    lib.concatMapStringsSep "\n" cmd sorted;
+
+
+  fileSystemsCfgFile =
+    let
+      mountable = lib.filterAttrs (_: value: hasDefinedMount value) datasets;
+    in
+    pkgs.runCommand "filesystem-config.nix"
+      {
+        buildInputs = with pkgs; [ jq nixpkgs-fmt ];
+        filesystems = builtins.toJSON {
+          fileSystems = lib.mapAttrs'
+            (
+              dataset: attrs:
+                {
+                  name = attrs.mount;
+                  value = {
+                    fsType = "zfs";
+                    device = "${dataset}";
+                  };
+                }
+            )
+            mountable;
+        };
+        passAsFile = [ "filesystems" ];
+      } ''
+      (
+        echo "builtins.fromJSON '''"
+        jq . < "$filesystemsPath"
+        echo "'''"
+      ) > $out
+
+      nixpkgs-fmt $out
+    '';
+
+  mergedConfig =
+    if configFile == null
+    then fileSystemsCfgFile
+    else
+      pkgs.runCommand "configuration.nix"
+        {
+          buildInputs = with pkgs; [ nixpkgs-fmt ];
+        }
+        ''
+          (
+            echo '{ imports = ['
+            printf "(%s)\n" "$(cat ${fileSystemsCfgFile})";
+            printf "(%s)\n" "$(cat ${configFile})";
+            echo ']; }'
+          ) > $out
+
+          nixpkgs-fmt $out
+        '';
+
+  image = (
+    pkgs.vmTools.override {
+      rootModules =
+        [ "zfs" "9p" "9pnet_virtio" "virtio_pci" "virtio_blk" ] ++
+        (pkgs.lib.optional pkgs.stdenv.hostPlatform.isx86 "rtc_cmos");
+      kernel = modulesTree;
+    }
+  ).runInLinuxVM (
+    pkgs.runCommand name
+      {
+        memSize = 1024;
+        QEMU_OPTS = "-drive file=$rootDiskImage,if=virtio,cache=unsafe,werror=report";
+        preVM = ''
+          PATH=$PATH:${pkgs.qemu_kvm}/bin
+          mkdir $out
+
+          rootDiskImage=root.raw
+          qemu-img create -f raw $rootDiskImage ${toString (bootSize + rootSize)}M
+        '';
+
+        postVM = ''
+            ${if formatOpt == "raw" then ''
+            mv $rootDiskImage $out/${rootFilename}
+          '' else ''
+            ${pkgs.qemu}/bin/qemu-img convert -f raw -O ${formatOpt} ${compress} $rootDiskImage $out/${rootFilename}
+          ''}
+            rootDiskImage=$out/${rootFilename}
+            set -x
+            ${postVM}
+        '';
+      } ''
+      export PATH=${tools}:$PATH
+      set -x
+
+      cp -sv /dev/vda /dev/sda
+      cp -sv /dev/vda /dev/xvda
+
+      parted --script /dev/vda -- \
+        mklabel gpt \
+        mkpart no-fs 1MiB 2MiB \
+        set 1 bios_grub on \
+        align-check optimal 1 \
+        mkpart primary fat32 2MiB ${toString bootSize}MiB \
+        align-check optimal 2 \
+        mkpart primary fat32 ${toString bootSize}MiB -1MiB \
+        align-check optimal 3 \
+        print
+
+      sfdisk --dump /dev/vda
+
+
+      zpool create \
+        ${stringifyProperties "  -o" rootPoolProperties} \
+        ${stringifyProperties "  -O" rootPoolFilesystemProperties} \
+        ${rootPoolName} /dev/vda3
+      parted --script /dev/vda -- print
+
+      ${createDatasets}
+      ${mountDatasets}
+
+      mkdir -p /mnt/boot
+      mkfs.vfat -n ESP /dev/vda2
+      mount /dev/vda2 /mnt/boot
+
+      mount
+
+      # Install a configuration.nix
+      mkdir -p /mnt/etc/nixos
+      # `cat` so it is mutable on the fs
+      cat ${mergedConfig} > /mnt/etc/nixos/configuration.nix
+
+      export NIX_STATE_DIR=$TMPDIR/state
+      nix-store --load-db < ${closureInfo}/registration
+
+      nixos-install \
+        --root /mnt \
+        --no-root-passwd \
+        --system ${config.system.build.toplevel} \
+        --substituters "" \
+        ${lib.optionalString includeChannel ''--channel ${channelSources}''}
+
+      df -h
+
+      umount /mnt/boot
+      ${unmountDatasets}
+
+      zpool export ${rootPoolName}
+    ''
+  );
+in
+image

From d3aff5fa3cb1ecd9d339e3801373ec3cca0057e4 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 8 Mar 2022 09:26:05 -0500
Subject: [PATCH 14/16] openstack-config: make the expandOnBoot option default
 to `all`

---
 nixos/modules/virtualisation/openstack-config.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nixos/modules/virtualisation/openstack-config.nix b/nixos/modules/virtualisation/openstack-config.nix
index c5c832fb8105..f7806228c653 100644
--- a/nixos/modules/virtualisation/openstack-config.nix
+++ b/nixos/modules/virtualisation/openstack-config.nix
@@ -50,7 +50,7 @@ in
       terminal_input console serial
     '';
 
-    services.zfs.expandOnBoot = mkIf cfg.zfs.enable "all";
+    services.zfs.expandOnBoot = mkIf cfg.zfs.enable (lib.mkDefault "all");
     boot.zfs.devNodes = mkIf cfg.zfs.enable "/dev/";
 
     # Allow root logins

From d99f30137492749eb011f01b5fcd3c29fd411825 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 8 Mar 2022 09:26:44 -0500
Subject: [PATCH 15/16] openstack-config: note the image metadata needed to
 boot a uefi image

---
 nixos/modules/virtualisation/openstack-config.nix | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/nixos/modules/virtualisation/openstack-config.nix b/nixos/modules/virtualisation/openstack-config.nix
index f7806228c653..af4f57466109 100644
--- a/nixos/modules/virtualisation/openstack-config.nix
+++ b/nixos/modules/virtualisation/openstack-config.nix
@@ -1,5 +1,8 @@
 { config, pkgs, lib, ... }:
 
+# image metadata:
+# hw_firmware_type=uefi
+
 let
   inherit (lib) mkIf mkDefault;
   cfg = config.openstack;

From 555bc5335bf8dcec8e0562ca1bedc96efdc8fc80 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 8 Mar 2022 09:29:00 -0500
Subject: [PATCH 16/16] openstack-image-zfs: start copying the channel now that
 we've mostly got the expressions down

---
 nixos/maintainers/scripts/openstack/openstack-image-zfs.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nixos/maintainers/scripts/openstack/openstack-image-zfs.nix b/nixos/maintainers/scripts/openstack/openstack-image-zfs.nix
index 515cc5cb38b3..d62a560642d0 100644
--- a/nixos/maintainers/scripts/openstack/openstack-image-zfs.nix
+++ b/nixos/maintainers/scripts/openstack/openstack-image-zfs.nix
@@ -3,7 +3,7 @@
 { config, lib, pkgs, ... }:
 let
   inherit (lib) mkOption types;
-  copyChannel = false;
+  copyChannel = true;
   cfg = config.openstackImage;
   imageBootMode = if config.openstack.efi then "uefi" else "legacy-bios";
 in