From 38495e1ec0399ab6b8873eee45d40cb0e525ed69 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Sun, 2 Jun 2024 00:12:11 +0100 Subject: [PATCH] nixos/prometheus: Added federation test case --- nixos/tests/prometheus/default.nix | 1 + nixos/tests/prometheus/federation.nix | 213 ++++++++++++++++++++++++++ 2 files changed, 214 insertions(+) create mode 100644 nixos/tests/prometheus/federation.nix diff --git a/nixos/tests/prometheus/default.nix b/nixos/tests/prometheus/default.nix index c14c6e188dd3..8c3dc16ddd19 100644 --- a/nixos/tests/prometheus/default.nix +++ b/nixos/tests/prometheus/default.nix @@ -4,6 +4,7 @@ }: { + federation = import ./federation.nix { inherit system pkgs; }; prometheus-pair = import ./prometheus-pair.nix { inherit system pkgs; }; pushgateway = import ./pushgateway.nix { inherit system pkgs; }; remote-write = import ./remote-write.nix { inherit system pkgs; }; diff --git a/nixos/tests/prometheus/federation.nix b/nixos/tests/prometheus/federation.nix new file mode 100644 index 000000000000..0f05166c8f5d --- /dev/null +++ b/nixos/tests/prometheus/federation.nix @@ -0,0 +1,213 @@ +import ../make-test-python.nix ({ lib, pkgs, ... }: + +{ + name = "prometheus-federation"; + + nodes = { + global1 = { config, pkgs, ... }: { + environment.systemPackages = [ pkgs.jq ]; + + networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; + + services.prometheus = { + enable = true; + globalConfig.scrape_interval = "2s"; + + scrapeConfigs = [ + { + job_name = "federate"; + honor_labels = true; + metrics_path = "/federate"; + + params = { + "match[]" = [ + "{job=\"node\"}" + "{job=\"prometheus\"}" + ]; + }; + + static_configs = [ + { + targets = [ + "prometheus1:${toString config.services.prometheus.port}" + "prometheus2:${toString config.services.prometheus.port}" + ]; + } + ]; + } + { + job_name = "prometheus"; + static_configs = [ + { + targets = [ + "global1:${toString config.services.prometheus.port}" + "global2:${toString config.services.prometheus.port}" + ]; + } + ]; + } + ]; + }; + }; + + global2 = { config, pkgs, ... }: { + environment.systemPackages = [ pkgs.jq ]; + + networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; + + services.prometheus = { + enable = true; + globalConfig.scrape_interval = "2s"; + + scrapeConfigs = [ + { + job_name = "federate"; + honor_labels = true; + metrics_path = "/federate"; + + params = { + "match[]" = [ + "{job=\"node\"}" + "{job=\"prometheus\"}" + ]; + }; + + static_configs = [ + { + targets = [ + "prometheus1:${toString config.services.prometheus.port}" + "prometheus2:${toString config.services.prometheus.port}" + ]; + } + ]; + } + { + job_name = "prometheus"; + static_configs = [ + { + targets = [ + "global1:${toString config.services.prometheus.port}" + "global2:${toString config.services.prometheus.port}" + ]; + } + ]; + } + ]; + }; + }; + + prometheus1 = { config, pkgs, ... }: { + environment.systemPackages = [ pkgs.jq ]; + + networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; + + services.prometheus = { + enable = true; + globalConfig.scrape_interval = "2s"; + + scrapeConfigs = [ + { + job_name = "node"; + static_configs = [ + { + targets = [ + "node1:${toString config.services.prometheus.exporters.node.port}" + ]; + } + ]; + } + { + job_name = "prometheus"; + static_configs = [ + { + targets = [ + "prometheus1:${toString config.services.prometheus.port}" + ]; + } + ]; + } + ]; + }; + }; + + prometheus2 = { config, pkgs, ... }: { + environment.systemPackages = [ pkgs.jq ]; + + networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ]; + + services.prometheus = { + enable = true; + globalConfig.scrape_interval = "2s"; + + scrapeConfigs = [ + { + job_name = "node"; + static_configs = [ + { + targets = [ + "node2:${toString config.services.prometheus.exporters.node.port}" + ]; + } + ]; + } + { + job_name = "prometheus"; + static_configs = [ + { + targets = [ + "prometheus2:${toString config.services.prometheus.port}" + ]; + } + ]; + } + ]; + }; + }; + + node1 = { config, pkgs, ... }: { + services.prometheus.exporters.node = { + enable = true; + openFirewall = true; + }; + }; + + node2 = { config, pkgs, ... }: { + services.prometheus.exporters.node = { + enable = true; + openFirewall = true; + }; + }; + }; + + testScript = '' + for machine in node1, node2: + machine.wait_for_unit("prometheus-node-exporter") + machine.wait_for_open_port(9100) + + for machine in prometheus1, prometheus2, global1, global2: + machine.wait_for_unit("prometheus") + machine.wait_for_open_port(9090) + + # Verify both servers got the same data from the exporter + for machine in prometheus1, prometheus2: + machine.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | " + + "jq '.data.result[0].value[1]' | grep '\"1\"'" + ) + machine.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | " + + "jq '.data.result[0].value[1]' | grep '\"1\"'" + ) + + for machine in global1, global2: + machine.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | " + + "jq '.data.result[0].value[1]' | grep '\"2\"'" + ) + + machine.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | " + + "jq '.data.result[0].value[1]' | grep '\"4\"'" + ) + ''; +})