nixos/iso-image: extremely cursed performance optimization for Hydra

Right now the worst case chain of events for building an ISO on Hydra is

    - copy everything to squashfs builder
    - run squashfs builder
    - download squashfs from builder
    - compress squashfs
    - upload squashfs to S3
    - copy squashfs to ISO builder
    - run ISO builder
    - download ISO from builder
    - compress ISO
    - upload ISO to S3

This inlines the squashfs build into the ISO build, which makes it

    - copy everything to ISO builder
    - run ISO builder
    - download ISO from builder
    - compress ISO
    - upload ISO to S3

Which should reduce queue runner load by $alot per ISO, which we have four of on small channels
(one release, one test per arch) and a lot more than four of on large channels (with various desktops)

nixos/lib/make-iso9660-image.nix

@@ -1,4 +1,4 @@
-{ stdenv, closureInfo, xorriso, syslinux, libossp_uuid
+{ lib, stdenv, callPackage, closureInfo, xorriso, syslinux, libossp_uuid, squashfsTools
 
 , # The file name of the resulting ISO image.
   isoName ? "cd.iso"
@@ -16,6 +16,17 @@
   # symlink to `object' that will be added to the CD.
   storeContents ? []
 
+, # In addition to `contents', the closure of the store paths listed
+  # in `squashfsContents' is compressed as squashfs and the result is
+  # placed in /nix-store.squashfs on the CD.
+  # FIXME: This is a performance optimization to avoid Hydra copying
+  # the squashfs between builders and should be removed when Hydra
+  # is smarter about scheduling.
+  squashfsContents ? []
+
+, # Compression settings for squashfs
+  squashfsCompression ? "xz -Xdict-size 100%"
+
 , # Whether this should be an El-Torito bootable CD.
   bootable ? false
 
@@ -45,12 +56,20 @@ assert bootable -> bootImage != "";
 assert efiBootable -> efiBootImage != "";
 assert usbBootable -> isohybridMbrImage != "";
 
+let
+  needSquashfs = squashfsContents != [];
+  makeSquashfsDrv = callPackage ./make-squashfs.nix {
+    storeContents = squashfsContents;
+    comp = squashfsCompression;
+  };
+in
 stdenv.mkDerivation {
   name = isoName;
   __structuredAttrs = true;
 
   buildCommandPath = ./make-iso9660-image.sh;
-  nativeBuildInputs = [ xorriso syslinux zstd libossp_uuid ];
+  nativeBuildInputs = [ xorriso syslinux zstd libossp_uuid ]
+    ++ lib.optionals needSquashfs makeSquashfsDrv.nativeBuildInputs;
 
   inherit isoName bootable bootImage compressImage volumeID efiBootImage efiBootable isohybridMbrImage usbBootable;
 
@@ -60,6 +79,8 @@ stdenv.mkDerivation {
   objects = map (x: x.object) storeContents;
   symlinks = map (x: x.symlink) storeContents;
 
+  squashfsCommand = lib.optionalString needSquashfs makeSquashfsDrv.buildCommand;
+
   # For obtaining the closure of `storeContents'.
   closureInfo = closureInfo { rootPaths = map (x: x.object) storeContents; };
 }

nixos/lib/make-iso9660-image.sh

@@ -68,6 +68,11 @@ for i in $(< $closureInfo/store-paths); do
     addPath "${i:1}" "$i"
 done
 
+# If needed, build a squashfs and add that
+if [[ -n "$squashfsCommand" ]]; then
+    (out="nix-store.squashfs" eval "$squashfsCommand")
+    addPath "nix-store.squashfs" "nix-store.squashfs"
+fi
 
 # Also include a manifest of the closures in a format suitable for
 # nix-store --load-db.

nixos/modules/installer/cd-dvd/iso-image.nix

@@ -811,12 +811,6 @@ in
       optional config.isoImage.includeSystemBuildDependencies
         config.system.build.toplevel.drvPath;
 
-    # Create the squashfs image that contains the Nix store.
-    system.build.squashfsStore = pkgs.callPackage ../../../lib/make-squashfs.nix {
-      storeContents = config.isoImage.storeContents;
-      comp = config.isoImage.squashfsCompression;
-    };
-
     # Individual files to be included on the CD, outside of the Nix
     # store on the CD.
     isoImage.contents =
@@ -827,9 +821,6 @@ in
         { source = config.system.build.initialRamdisk + "/" + config.system.boot.loader.initrdFile;
           target = "/boot/" + config.system.boot.loader.initrdFile;
         }
-        { source = config.system.build.squashfsStore;
-          target = "/nix-store.squashfs";
-        }
         { source = pkgs.writeText "version" config.system.nixos.label;
           target = "/version.txt";
         }
@@ -878,6 +869,8 @@ in
       bootable = config.isoImage.makeBiosBootable;
       bootImage = "/isolinux/isolinux.bin";
       syslinux = if config.isoImage.makeBiosBootable then pkgs.syslinux else null;
+      squashfsContents = config.isoImage.storeContents;
+      squashfsCompression = config.isoImage.squashfsCompression;
     } // optionalAttrs (config.isoImage.makeUsbBootable && config.isoImage.makeBiosBootable) {
       usbBootable = true;
       isohybridMbrImage = "${pkgs.syslinux}/share/syslinux/isohdpfx.bin";