nixpkgs/nixos/tests/spark/default.nix

{ pkgs, ... }:

let
  inherit (pkgs) lib;
  tests = {
    default = testsForPackage { sparkPackage = pkgs.spark; };
  };

  testsForPackage = args: lib.recurseIntoAttrs {
    sparkCluster = testSparkCluster args;
    passthru.override = args': testsForPackage (args // args');
  };
  testSparkCluster = { sparkPackage, ... }: pkgs.nixosTest ({
    name = "spark";

    nodes = {
      worker = { nodes, pkgs, ... }: {
        services.spark = {
          package = sparkPackage;
          worker = {
            enable = true;
            master = "master:7077";
          };
        };
        virtualisation.memorySize = 2048;
      };
      master = { config, pkgs, ... }: {
        services.spark = {
          package = sparkPackage;
          master = {
            enable = true;
            bind = "0.0.0.0";
          };
        };
        networking.firewall.allowedTCPPorts = [ 22 7077 8080 ];
      };
    };

    testScript = ''
      master.wait_for_unit("spark-master.service")
      worker.wait_for_unit("spark-worker.service")
      worker.copy_from_host( "${./spark_sample.py}", "/spark_sample.py" )
      assert "<title>Spark Master at spark://" in worker.succeed("curl -sSfkL http://master:8080/")
      worker.succeed("spark-submit --master spark://master:7077 --executor-memory 512m --executor-cores 1 /spark_sample.py")
    '';
  });
in tests