nixpkgs/nixos/tests/iscsi-multipath-root.nix

import ./make-test-python.nix (
  { pkgs, lib, ... }:
  let
    initiatorName = "iqn.2020-08.org.linux-iscsi.initiatorhost:example";
    targetName = "iqn.2003-01.org.linux-iscsi.target.x8664:sn.acf8fd9c23af";
  in
  {
    name = "iscsi";
    meta = {
      maintainers = pkgs.lib.teams.deshaw.members;
    };

    nodes = {
      target = { config, pkgs, lib, ... }: {
        virtualisation.vlans = [ 1 2 ];
        services.target = {
          enable = true;
          config = {
            fabric_modules = [ ];
            storage_objects = [
              {
                dev = "/dev/vdb";
                name = "test";
                plugin = "block";
                write_back = true;
                wwn = "92b17c3f-6b40-4168-b082-ceeb7b495522";
              }
            ];
            targets = [
              {
                fabric = "iscsi";
                tpgs = [
                  {
                    enable = true;
                    attributes = {
                      authentication = 0;
                      generate_node_acls = 1;
                    };
                    luns = [
                      {
                        alias = "94dfe06967";
                        alua_tg_pt_gp_name = "default_tg_pt_gp";
                        index = 0;
                        storage_object = "/backstores/block/test";
                      }
                    ];
                    node_acls = [
                      {
                        mapped_luns = [
                          {
                            alias = "d42f5bdf8a";
                            index = 0;
                            tpg_lun = 0;
                            write_protect = false;
                          }
                        ];
                        node_wwn = initiatorName;
                      }
                    ];
                    portals = [
                      {
                        ip_address = "0.0.0.0";
                        iser = false;
                        offload = false;
                        port = 3260;
                      }
                    ];
                    tag = 1;
                  }
                ];
                wwn = targetName;
              }
            ];
          };
        };

        networking.firewall.allowedTCPPorts = [ 3260 ];
        networking.firewall.allowedUDPPorts = [ 3260 ];

        virtualisation.memorySize = 2048;
        virtualisation.emptyDiskImages = [ 2048 ];
      };

      initiatorAuto = { nodes, config, pkgs, ... }: {
        virtualisation.vlans = [ 1 2 ];

        services.multipath = {
          enable = true;
          defaults = ''
            find_multipaths yes
            user_friendly_names yes
          '';
          pathGroups = [
            {
              alias = 123456;
              wwid = "3600140592b17c3f6b404168b082ceeb7";
            }
          ];
        };

        services.openiscsi = {
          enable = true;
          enableAutoLoginOut = true;
          discoverPortal = "target";
          name = initiatorName;
        };

        environment.systemPackages = with pkgs; [
          xfsprogs
        ];

        environment.etc."initiator-root-disk-closure".source = nodes.initiatorRootDisk.config.system.build.toplevel;

        nix.binaryCaches = lib.mkForce [ ];
        nix.extraOptions = ''
          hashed-mirrors =
          connect-timeout = 1
        '';
      };

      initiatorRootDisk = { config, pkgs, modulesPath, lib, ... }: {
        boot.initrd.network.enable = true;
        boot.loader.grub.enable = false;

        boot.kernelParams = lib.mkOverride 5 (
          [
            "boot.shell_on_fail"
            "console=tty1"
            "ip=192.168.1.1:::255.255.255.0::ens9:none"
            "ip=192.168.2.1:::255.255.255.0::ens10:none"
          ]
        );

        # defaults to true, puts some code in the initrd that tries to mount an overlayfs on /nix/store
        virtualisation.writableStore = false;
        virtualisation.vlans = [ 1 2 ];

        services.multipath = {
          enable = true;
          defaults = ''
            find_multipaths yes
            user_friendly_names yes
          '';
          pathGroups = [
            {
              alias = 123456;
              wwid = "3600140592b17c3f6b404168b082ceeb7";
            }
          ];
        };

        fileSystems = lib.mkOverride 5 {
          "/" = {
            fsType = "xfs";
            device = "/dev/mapper/123456";
            options = [ "_netdev" ];
          };
        };

        boot.initrd.extraFiles."etc/multipath/wwids".source = pkgs.writeText "wwids" "/3600140592b17c3f6b404168b082ceeb7/";

        boot.iscsi-initiator = {
          discoverPortal = "target";
          name = initiatorName;
          target = targetName;
          extraIscsiCommands = ''
            iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login
          '';
        };
      };

    };

    testScript = { nodes, ... }: ''
      target.start()
      target.wait_for_unit("iscsi-target.service")

      initiatorAuto.start()

      initiatorAuto.wait_for_unit("iscsid.service")
      initiatorAuto.wait_for_unit("iscsi.service")
      initiatorAuto.get_unit_info("iscsi")

      # Expecting this to fail since we should already know about 192.168.1.3
      initiatorAuto.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
      # Expecting this to succeed since we don't yet know about 192.168.2.3
      initiatorAuto.succeed("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")

      # /dev/sda is provided by iscsi on target
      initiatorAuto.succeed("set -x; while ! test -e /dev/sda; do sleep 1; done")

      initiatorAuto.succeed("mkfs.xfs /dev/sda")
      initiatorAuto.succeed("mkdir /mnt")

      # Start by verifying /dev/sda and /dev/sdb are both the same disk
      initiatorAuto.succeed("mount /dev/sda /mnt")
      initiatorAuto.succeed("touch /mnt/hi")
      initiatorAuto.succeed("umount /mnt")

      initiatorAuto.succeed("mount /dev/sdb /mnt")
      initiatorAuto.succeed("test -e /mnt/hi")
      initiatorAuto.succeed("umount /mnt")

      initiatorAuto.succeed("systemctl restart multipathd")
      initiatorAuto.succeed("multipath -ll | systemd-cat")

      # Install our RootDisk machine to 123456, the alias to the device that multipath is now managing
      initiatorAuto.succeed("mount /dev/mapper/123456 /mnt")
      initiatorAuto.succeed("mkdir -p /mnt/etc/{multipath,iscsi}")
      initiatorAuto.succeed("cp -r /etc/multipath/wwids /mnt/etc/multipath/wwids")
      initiatorAuto.succeed("cp -r /etc/iscsi/{nodes,send_targets} /mnt/etc/iscsi")
      initiatorAuto.succeed(
        "nixos-install --no-bootloader --no-root-passwd --system /etc/initiator-root-disk-closure"
      )
      initiatorAuto.succeed("umount /mnt")
      initiatorAuto.shutdown()

      initiatorRootDisk.start()
      initiatorRootDisk.wait_for_unit("multi-user.target")
      initiatorRootDisk.wait_for_unit("iscsid")

      # Log in over both nodes
      initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
      initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
      initiatorRootDisk.succeed("systemctl restart multipathd")
      initiatorRootDisk.succeed("multipath -ll | systemd-cat")

      # Verify we can write and sync the root disk
      initiatorRootDisk.succeed("mkdir /scratch")
      initiatorRootDisk.succeed("touch /scratch/both-up")
      initiatorRootDisk.succeed("sync /scratch")

      # Verify we can write to the root with ens9 (sda, 192.168.1.3) down
      initiatorRootDisk.succeed("ip link set ens9 down")
      initiatorRootDisk.succeed("touch /scratch/ens9-down")
      initiatorRootDisk.succeed("sync /scratch")
      initiatorRootDisk.succeed("ip link set ens9 up")

      # todo: better way to wait until multipath notices the link is back
      initiatorRootDisk.succeed("sleep 5")
      initiatorRootDisk.succeed("touch /scratch/both-down")
      initiatorRootDisk.succeed("sync /scratch")

      # Verify we can write to the root with ens10 (sdb, 192.168.2.3) down
      initiatorRootDisk.succeed("ip link set ens10 down")
      initiatorRootDisk.succeed("touch /scratch/ens10-down")
      initiatorRootDisk.succeed("sync /scratch")
      initiatorRootDisk.succeed("ip link set ens10 up")
      initiatorRootDisk.succeed("touch /scratch/ens10-down")
      initiatorRootDisk.succeed("sync /scratch")

      initiatorRootDisk.succeed("ip link set ens9 up")
      initiatorRootDisk.succeed("ip link set ens10 up")
      initiatorRootDisk.shutdown()

      # Verify we can boot with the target's eth1 down, forcing
      # it to multipath via the second link
      target.succeed("ip link set eth1 down")
      initiatorRootDisk.start()
      initiatorRootDisk.wait_for_unit("multi-user.target")
      initiatorRootDisk.wait_for_unit("iscsid")
      initiatorRootDisk.succeed("test -e /scratch/both-up")
    '';
  }
)