mirror of
https://github.com/NixOS/nixpkgs.git
synced 2024-11-22 15:03:28 +00:00
spark: add passthru test
This commit is contained in:
parent
fc7aceed42
commit
392bc54225
@ -1,28 +1,47 @@
|
||||
import ../make-test-python.nix ({...}: {
|
||||
name = "spark";
|
||||
{ pkgs, ... }:
|
||||
|
||||
nodes = {
|
||||
worker = { nodes, pkgs, ... }: {
|
||||
services.spark.worker = {
|
||||
enable = true;
|
||||
master = "master:7077";
|
||||
};
|
||||
virtualisation.memorySize = 2048;
|
||||
};
|
||||
master = { config, pkgs, ... }: {
|
||||
services.spark.master = {
|
||||
enable = true;
|
||||
bind = "0.0.0.0";
|
||||
};
|
||||
networking.firewall.allowedTCPPorts = [ 22 7077 8080 ];
|
||||
};
|
||||
let
|
||||
inherit (pkgs) lib;
|
||||
tests = {
|
||||
default = testsForPackage { sparkPackage = pkgs.spark; };
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
master.wait_for_unit("spark-master.service")
|
||||
worker.wait_for_unit("spark-worker.service")
|
||||
worker.copy_from_host( "${./spark_sample.py}", "/spark_sample.py" )
|
||||
assert "<title>Spark Master at spark://" in worker.succeed("curl -sSfkL http://master:8080/")
|
||||
worker.succeed("spark-submit --master spark://master:7077 --executor-memory 512m --executor-cores 1 /spark_sample.py")
|
||||
'';
|
||||
})
|
||||
testsForPackage = args: lib.recurseIntoAttrs {
|
||||
sparkCluster = testSparkCluster args;
|
||||
passthru.override = args': testsForPackage (args // args');
|
||||
};
|
||||
testSparkCluster = { sparkPackage, ... }: pkgs.nixosTest ({
|
||||
name = "spark";
|
||||
|
||||
nodes = {
|
||||
worker = { nodes, pkgs, ... }: {
|
||||
services.spark = {
|
||||
package = sparkPackage;
|
||||
worker = {
|
||||
enable = true;
|
||||
master = "master:7077";
|
||||
};
|
||||
};
|
||||
virtualisation.memorySize = 2048;
|
||||
};
|
||||
master = { config, pkgs, ... }: {
|
||||
services.spark = {
|
||||
package = sparkPackage;
|
||||
master = {
|
||||
enable = true;
|
||||
bind = "0.0.0.0";
|
||||
};
|
||||
};
|
||||
networking.firewall.allowedTCPPorts = [ 22 7077 8080 ];
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
master.wait_for_unit("spark-master.service")
|
||||
worker.wait_for_unit("spark-worker.service")
|
||||
worker.copy_from_host( "${./spark_sample.py}", "/spark_sample.py" )
|
||||
assert "<title>Spark Master at spark://" in worker.succeed("curl -sSfkL http://master:8080/")
|
||||
worker.succeed("spark-submit --master spark://master:7077 --executor-memory 512m --executor-cores 1 /spark_sample.py")
|
||||
'';
|
||||
});
|
||||
in tests
|
||||
|
@ -10,6 +10,7 @@
|
||||
, hadoop
|
||||
, RSupport ? true
|
||||
, R
|
||||
, nixosTests
|
||||
}:
|
||||
|
||||
let
|
||||
@ -60,6 +61,10 @@ let
|
||||
ln -s $out/lib/${untarDir}/lib/spark-assembly-*.jar $out/share/java
|
||||
'';
|
||||
|
||||
passthru.tests = nixosTests.spark.default.passthru.override {
|
||||
sparkPackage = finalAttrs.finalPackage;
|
||||
};
|
||||
|
||||
meta = {
|
||||
description = "Apache Spark is a fast and general engine for large-scale data processing";
|
||||
homepage = "https://spark.apache.org/";
|
||||
|
Loading…
Reference in New Issue
Block a user