spark: add passthru test

This commit is contained in:
illustris 2023-10-22 18:16:20 +05:30
parent fc7aceed42
commit 392bc54225
No known key found for this signature in database
GPG Key ID: 56C8FC0B899FEFA3
2 changed files with 49 additions and 25 deletions

View File

@ -1,28 +1,47 @@
import ../make-test-python.nix ({...}: {
name = "spark";
{ pkgs, ... }:
nodes = {
worker = { nodes, pkgs, ... }: {
services.spark.worker = {
enable = true;
master = "master:7077";
};
virtualisation.memorySize = 2048;
};
master = { config, pkgs, ... }: {
services.spark.master = {
enable = true;
bind = "0.0.0.0";
};
networking.firewall.allowedTCPPorts = [ 22 7077 8080 ];
};
let
inherit (pkgs) lib;
tests = {
default = testsForPackage { sparkPackage = pkgs.spark; };
};
testScript = ''
master.wait_for_unit("spark-master.service")
worker.wait_for_unit("spark-worker.service")
worker.copy_from_host( "${./spark_sample.py}", "/spark_sample.py" )
assert "<title>Spark Master at spark://" in worker.succeed("curl -sSfkL http://master:8080/")
worker.succeed("spark-submit --master spark://master:7077 --executor-memory 512m --executor-cores 1 /spark_sample.py")
'';
})
testsForPackage = args: lib.recurseIntoAttrs {
sparkCluster = testSparkCluster args;
passthru.override = args': testsForPackage (args // args');
};
testSparkCluster = { sparkPackage, ... }: pkgs.nixosTest ({
name = "spark";
nodes = {
worker = { nodes, pkgs, ... }: {
services.spark = {
package = sparkPackage;
worker = {
enable = true;
master = "master:7077";
};
};
virtualisation.memorySize = 2048;
};
master = { config, pkgs, ... }: {
services.spark = {
package = sparkPackage;
master = {
enable = true;
bind = "0.0.0.0";
};
};
networking.firewall.allowedTCPPorts = [ 22 7077 8080 ];
};
};
testScript = ''
master.wait_for_unit("spark-master.service")
worker.wait_for_unit("spark-worker.service")
worker.copy_from_host( "${./spark_sample.py}", "/spark_sample.py" )
assert "<title>Spark Master at spark://" in worker.succeed("curl -sSfkL http://master:8080/")
worker.succeed("spark-submit --master spark://master:7077 --executor-memory 512m --executor-cores 1 /spark_sample.py")
'';
});
in tests

View File

@ -10,6 +10,7 @@
, hadoop
, RSupport ? true
, R
, nixosTests
}:
let
@ -60,6 +61,10 @@ let
ln -s $out/lib/${untarDir}/lib/spark-assembly-*.jar $out/share/java
'';
passthru.tests = nixosTests.spark.default.passthru.override {
sparkPackage = finalAttrs.finalPackage;
};
meta = {
description = "Apache Spark is a fast and general engine for large-scale data processing";
homepage = "https://spark.apache.org/";