nixpkgs/nixos/tests/prometheus-exporters.nix
Ivan Trubach 6008ed89f6 nixos/prometheus.exporters.pgbouncer: do not assume that pgbouncer runs on localhost
PgBouncer instance running on localhost may not be the on being
monitored in connectionString. Remove checks that forbid valid
configuration from being used and instead document requirements for
PgBouncer configuration when used with the exporter.
2024-08-08 05:54:45 +03:00

1799 lines
54 KiB
Nix

{ system ? builtins.currentSystem
, config ? { }
, pkgs ? import ../.. { inherit system config; }
}:
let
inherit (import ../lib/testing-python.nix { inherit system pkgs; }) makeTest;
inherit (pkgs.lib) concatStringsSep maintainers mapAttrs mkMerge
removeSuffix replaceStrings singleton splitString makeBinPath;
/*
* The attrset `exporterTests` contains one attribute
* for each exporter test. Each of these attributes
* is expected to be an attrset containing:
*
* `exporterConfig`:
* this attribute set contains config for the exporter itself
*
* `exporterTest`
* this attribute set contains test instructions
*
* `metricProvider` (optional)
* this attribute contains additional machine config
*
* `nodeName` (optional)
* override an incompatible testnode name
*
* Example:
* exporterTests.<exporterName> = {
* exporterConfig = {
* enable = true;
* };
* metricProvider = {
* services.<metricProvider>.enable = true;
* };
* exporterTest = ''
* wait_for_unit("prometheus-<exporterName>-exporter.service")
* wait_for_open_port(1234)
* succeed("curl -sSf 'localhost:1234/metrics'")
* '';
* };
*
* # this would generate the following test config:
*
* nodes.<exporterName> = {
* services.prometheus.<exporterName> = {
* enable = true;
* };
* services.<metricProvider>.enable = true;
* };
*
* testScript = ''
* <exporterName>.start()
* <exporterName>.wait_for_unit("prometheus-<exporterName>-exporter.service")
* <exporterName>.wait_for_open_port(1234)
* <exporterName>.succeed("curl -sSf 'localhost:1234/metrics'")
* <exporterName>.shutdown()
* '';
*/
exporterTests = {
apcupsd = {
exporterConfig = {
enable = true;
};
metricProvider = {
services.apcupsd.enable = true;
};
exporterTest = ''
wait_for_unit("apcupsd.service")
wait_for_open_port(3551)
wait_for_unit("prometheus-apcupsd-exporter.service")
wait_for_open_port(9162)
succeed("curl -sSf http://localhost:9162/metrics | grep 'apcupsd_info'")
'';
};
artifactory = {
exporterConfig = {
enable = true;
artiUsername = "artifactory-username";
artiPassword = "artifactory-password";
};
exporterTest = ''
wait_for_unit("prometheus-artifactory-exporter.service")
wait_for_open_port(9531)
succeed(
"curl -sSf http://localhost:9531/metrics | grep 'artifactory_up'"
)
'';
};
bind = {
exporterConfig = {
enable = true;
};
metricProvider = {
services.bind.enable = true;
services.bind.extraConfig = ''
statistics-channels {
inet 127.0.0.1 port 8053 allow { localhost; };
};
'';
};
exporterTest = ''
wait_for_unit("prometheus-bind-exporter.service")
wait_for_open_port(9119)
succeed(
"curl -sSf http://localhost:9119/metrics | grep 'bind_query_recursions_total 0'"
)
'';
};
bird = {
exporterConfig = {
enable = true;
};
metricProvider = {
services.bird2.enable = true;
services.bird2.config = ''
router id 127.0.0.1;
protocol kernel MyObviousTestString {
ipv4 {
import all;
export none;
};
}
protocol device {
}
'';
};
exporterTest = ''
wait_for_unit("prometheus-bird-exporter.service")
wait_for_open_port(9324)
wait_until_succeeds(
"curl -sSf http://localhost:9324/metrics | grep 'MyObviousTestString'"
)
'';
};
bitcoin = {
exporterConfig = {
enable = true;
rpcUser = "bitcoinrpc";
rpcPasswordFile = pkgs.writeText "password" "hunter2";
};
metricProvider = {
services.bitcoind.default.enable = true;
services.bitcoind.default.rpc.users.bitcoinrpc.passwordHMAC = "e8fe33f797e698ac258c16c8d7aadfbe$872bdb8f4d787367c26bcfd75e6c23c4f19d44a69f5d1ad329e5adf3f82710f7";
};
exporterTest = ''
wait_for_unit("prometheus-bitcoin-exporter.service")
wait_for_unit("bitcoind-default.service")
wait_for_open_port(9332)
succeed("curl -sSf http://localhost:9332/metrics | grep '^bitcoin_blocks '")
'';
};
blackbox = {
exporterConfig = {
enable = true;
configFile = pkgs.writeText "config.yml" (builtins.toJSON {
modules.icmp_v6 = {
prober = "icmp";
icmp.preferred_ip_protocol = "ip6";
};
});
};
exporterTest = ''
wait_for_unit("prometheus-blackbox-exporter.service")
wait_for_open_port(9115)
succeed(
"curl -sSf 'http://localhost:9115/probe?target=localhost&module=icmp_v6' | grep 'probe_success 1'"
)
'';
};
borgmatic = {
exporterConfig = {
enable = true;
user = "root";
};
metricProvider = {
services.borgmatic.enable = true;
services.borgmatic.settings.source_directories = [ "/home" ];
services.borgmatic.settings.repositories = [ { label = "local"; path = "/var/backup"; } ];
services.borgmatic.settings.keep_daily = 10;
};
exporterTest = ''
succeed("borgmatic rcreate -e none")
succeed("borgmatic")
wait_for_unit("prometheus-borgmatic-exporter.service")
wait_for_open_port(9996)
succeed("curl -sSf localhost:9996/metrics | grep 'borg_total_backups{repository=\"/var/backup\"} 1'")
'';
};
collectd = {
exporterConfig = {
enable = true;
extraFlags = [ "--web.collectd-push-path /collectd" ];
};
exporterTest = let postData = replaceStrings [ "\n" ] [ "" ] ''
[{
"values":[23],
"dstypes":["gauge"],
"type":"gauge",
"interval":1000,
"host":"testhost",
"plugin":"testplugin",
"time":DATE
}]
''; in
''
wait_for_unit("prometheus-collectd-exporter.service")
wait_for_open_port(9103)
succeed(
'echo \'${postData}\'> /tmp/data.json'
)
succeed('sed -ie "s DATE $(date +%s) " /tmp/data.json')
succeed(
"curl -sSfH 'Content-Type: application/json' -X POST --data @/tmp/data.json localhost:9103/collectd"
)
succeed(
"curl -sSf localhost:9103/metrics | grep 'collectd_testplugin_gauge{instance=\"testhost\"} 23'"
)
'';
};
deluge = {
exporterConfig = {
enable = true;
port = 1234;
listenAddress = "127.0.0.1";
delugeUser = "user";
delugePort = 2345;
delugePasswordFile = pkgs.writeText "password" "weak_password";
};
metricProvider = {
services.deluge.enable = true;
services.deluge.declarative = true;
services.deluge.config.daemon_port = 2345;
services.deluge.authFile = pkgs.writeText "authFile" ''
localclient:abcdef:10
user:weak_password:10
'';
};
exporterTest = ''
wait_for_unit("deluged.service")
wait_for_open_port(2345)
wait_for_unit("prometheus-deluge-exporter.service")
wait_for_open_port(1234)
succeed("curl -sSf http://localhost:1234 | grep 'deluge_torrents'")
'';
};
dnsmasq = {
exporterConfig = {
enable = true;
leasesPath = "/var/lib/dnsmasq/dnsmasq.leases";
};
metricProvider = {
services.dnsmasq.enable = true;
};
exporterTest = ''
wait_for_unit("dnsmasq.service")
wait_for_open_port(53)
wait_for_file("/var/lib/dnsmasq/dnsmasq.leases")
wait_for_unit("prometheus-dnsmasq-exporter.service")
wait_for_open_port(9153)
succeed("curl -sSf http://localhost:9153/metrics | grep 'dnsmasq_leases 0'")
'';
};
dnssec = {
exporterConfig = {
enable = true;
configuration = {
records = [
{
zone = "example.com";
record = "@";
type = "SOA";
}
];
};
resolvers = [ "127.0.0.1:53" ];
};
metricProvider = {
services.knot = {
enable = true;
settingsFile = pkgs.writeText "knot.conf" ''
server:
listen: 127.0.0.1@53
template:
- id: default
storage: ${pkgs.buildEnv {
name = "zones";
paths = [(pkgs.writeTextDir "example.com.zone" ''
@ SOA ns1.example.com. noc.example.com. 2024032401 86400 7200 3600000 172800
@ NS ns1
ns1 A 192.168.0.1
'')];
}}
zonefile-load: difference
zonefile-sync: -1
zone:
- domain: example.com
file: example.com.zone
dnssec-signing: on
'';
};
};
exporterTest = ''
wait_for_unit("knot.service")
wait_for_open_port(53)
wait_for_unit("prometheus-dnssec-exporter.service")
wait_for_open_port(9204)
succeed("curl -sSf http://localhost:9204/metrics | grep 'example.com'")
'';
};
# Access to WHOIS server is required to properly test this exporter, so
# just perform basic sanity check that the exporter is running and returns
# a failure.
domain = {
exporterConfig = {
enable = true;
};
exporterTest = ''
wait_for_unit("prometheus-domain-exporter.service")
wait_for_open_port(9222)
succeed("curl -sSf 'http://localhost:9222/probe?target=nixos.org'")
'';
};
dovecot = {
exporterConfig = {
enable = true;
scopes = [ "global" ];
socketPath = "/var/run/dovecot2/old-stats";
user = "root"; # <- don't use user root in production
};
metricProvider = {
services.dovecot2.enable = true;
};
exporterTest = ''
wait_for_unit("prometheus-dovecot-exporter.service")
wait_for_open_port(9166)
succeed(
"curl -sSf http://localhost:9166/metrics | grep 'dovecot_up{scope=\"global\"} 1'"
)
'';
};
fastly = {
exporterConfig = {
enable = true;
tokenPath = pkgs.writeText "token" "abc123";
};
exporterTest = ''
wait_for_unit("prometheus-fastly-exporter.service")
wait_for_open_port(9118)
'';
};
fritzbox = {
# TODO add proper test case
exporterConfig = {
enable = true;
};
exporterTest = ''
wait_for_unit("prometheus-fritzbox-exporter.service")
wait_for_open_port(9133)
succeed(
"curl -sSf http://localhost:9133/metrics | grep 'fritzbox_exporter_collect_errors 0'"
)
'';
};
graphite = {
exporterConfig = {
enable = true;
port = 9108;
graphitePort = 9109;
mappingSettings.mappings = [{
match = "test.*.*";
name = "testing";
labels = {
protocol = "$1";
author = "$2";
};
}];
};
exporterTest = ''
wait_for_unit("prometheus-graphite-exporter.service")
wait_for_open_port(9108)
wait_for_open_port(9109)
succeed("echo test.tcp.foo-bar 1234 $(date +%s) | nc -w1 localhost 9109")
succeed("curl -sSf http://localhost:9108/metrics | grep 'testing{author=\"foo-bar\",protocol=\"tcp\"} 1234'")
'';
};
idrac = {
exporterConfig = {
enable = true;
port = 9348;
configuration = {
hosts = {
default = { username = "username"; password = "password"; };
};
};
};
exporterTest = ''
wait_for_unit("prometheus-idrac-exporter.service")
wait_for_open_port(9348)
wait_until_succeeds("curl localhost:9348")
'';
};
influxdb = {
exporterConfig = {
enable = true;
sampleExpiry = "3s";
};
exporterTest = ''
wait_for_unit("prometheus-influxdb-exporter.service")
wait_for_open_port(9122)
succeed(
"curl -XPOST http://localhost:9122/write --data-binary 'influxdb_exporter,distro=nixos,added_in=21.09 value=1'"
)
succeed(
"curl -sSf http://localhost:9122/metrics | grep 'nixos'"
)
execute("sleep 5")
fail(
"curl -sSf http://localhost:9122/metrics | grep 'nixos'"
)
'';
};
ipmi = {
exporterConfig = {
enable = true;
};
exporterTest = ''
wait_for_unit("prometheus-ipmi-exporter.service")
wait_for_open_port(9290)
succeed(
"curl -sSf http://localhost:9290/metrics | grep 'ipmi_scrape_duration_seconds'"
)
'';
};
jitsi = {
exporterConfig = {
enable = true;
};
metricProvider = {
systemd.services.prometheus-jitsi-exporter.after = [ "jitsi-videobridge2.service" ];
services.jitsi-videobridge = {
enable = true;
colibriRestApi = true;
};
};
exporterTest = ''
wait_for_unit("jitsi-videobridge2.service")
wait_for_open_port(8080)
wait_for_unit("prometheus-jitsi-exporter.service")
wait_for_open_port(9700)
wait_until_succeeds(
'journalctl -eu prometheus-jitsi-exporter.service -o cat | grep "key=participants"'
)
succeed("curl -sSf 'localhost:9700/metrics' | grep 'jitsi_participants 0'")
'';
};
json = {
exporterConfig = {
enable = true;
url = "http://localhost";
configFile = pkgs.writeText "json-exporter-conf.json" (builtins.toJSON {
modules = {
default = {
metrics = [
{ name = "json_test_metric"; path = "{ .test }"; }
];
};
};
});
};
metricProvider = {
systemd.services.prometheus-json-exporter.after = [ "nginx.service" ];
services.nginx = {
enable = true;
virtualHosts.localhost.locations."/".extraConfig = ''
return 200 "{\"test\":1}";
'';
};
};
exporterTest = ''
wait_for_unit("nginx.service")
wait_for_open_port(80)
wait_for_unit("prometheus-json-exporter.service")
wait_for_open_port(7979)
succeed(
"curl -sSf 'localhost:7979/probe?target=http://localhost' | grep 'json_test_metric 1'"
)
'';
};
knot = {
exporterConfig = {
enable = true;
};
metricProvider = {
services.knot = {
enable = true;
extraArgs = [ "-v" ];
settingsFile = pkgs.writeText "knot.conf" ''
server:
listen: 127.0.0.1@53
template:
- id: default
global-module: mod-stats
dnssec-signing: off
zonefile-sync: -1
journal-db: /var/lib/knot/journal
kasp-db: /var/lib/knot/kasp
timer-db: /var/lib/knot/timer
zonefile-load: difference
storage: ${pkgs.buildEnv {
name = "foo";
paths = [
(pkgs.writeTextDir "test.zone" ''
@ SOA ns.example.com. noc.example.com. 2019031301 86400 7200 3600000 172800
@ NS ns1
@ NS ns2
ns1 A 192.168.0.1
'')
];
}}
mod-stats:
- id: custom
edns-presence: on
query-type: on
zone:
- domain: test
file: test.zone
module: mod-stats/custom
'';
};
};
exporterTest = ''
wait_for_unit("knot.service")
wait_for_unit("prometheus-knot-exporter.service")
wait_for_open_port(9433)
succeed("curl -sSf 'localhost:9433' | grep '2\.019031301'")
'';
};
keylight = {
# A hardware device is required to properly test this exporter, so just
# perform a couple of basic sanity checks that the exporter is running
# and requires a target, but cannot reach a specified target.
exporterConfig = {
enable = true;
};
exporterTest = ''
wait_for_unit("prometheus-keylight-exporter.service")
wait_for_open_port(9288)
succeed(
"curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics | grep '400'"
)
succeed(
"curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics?target=nosuchdevice | grep '500'"
)
'';
};
lnd = {
exporterConfig = {
enable = true;
lndTlsPath = "/var/lib/lnd/tls.cert";
lndMacaroonDir = "/var/lib/lnd";
extraFlags = [ "--lnd.network=regtest" ];
};
metricProvider = {
systemd.services.prometheus-lnd-exporter.serviceConfig.RestartSec = 15;
systemd.services.prometheus-lnd-exporter.after = [ "lnd.service" ];
services.bitcoind.regtest = {
enable = true;
extraConfig = ''
rpcauth=bitcoinrpc:e8fe33f797e698ac258c16c8d7aadfbe$872bdb8f4d787367c26bcfd75e6c23c4f19d44a69f5d1ad329e5adf3f82710f7
zmqpubrawblock=tcp://127.0.0.1:28332
zmqpubrawtx=tcp://127.0.0.1:28333
'';
extraCmdlineOptions = [ "-regtest" ];
};
systemd.services.lnd = {
serviceConfig.ExecStart = ''
${pkgs.lnd}/bin/lnd \
--datadir=/var/lib/lnd \
--tlscertpath=/var/lib/lnd/tls.cert \
--tlskeypath=/var/lib/lnd/tls.key \
--logdir=/var/log/lnd \
--bitcoin.active \
--bitcoin.regtest \
--bitcoin.node=bitcoind \
--bitcoind.rpcuser=bitcoinrpc \
--bitcoind.rpcpass=hunter2 \
--bitcoind.zmqpubrawblock=tcp://127.0.0.1:28332 \
--bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333 \
--readonlymacaroonpath=/var/lib/lnd/readonly.macaroon
'';
serviceConfig.StateDirectory = "lnd";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
};
# initialize wallet, creates macaroon needed by exporter
systemd.services.lnd.postStart = ''
${pkgs.curl}/bin/curl \
--retry 20 \
--retry-delay 1 \
--retry-connrefused \
--cacert /var/lib/lnd/tls.cert \
-X GET \
https://localhost:8080/v1/genseed | ${pkgs.jq}/bin/jq -c '.cipher_seed_mnemonic' > /tmp/seed
${pkgs.curl}/bin/curl \
--retry 20 \
--retry-delay 1 \
--retry-connrefused \
--cacert /var/lib/lnd/tls.cert \
-X POST \
-d "{\"wallet_password\": \"asdfasdfasdf\", \"cipher_seed_mnemonic\": $(cat /tmp/seed | tr -d '\n')}" \
https://localhost:8080/v1/initwallet
'';
};
exporterTest = ''
wait_for_unit("lnd.service")
wait_for_open_port(10009)
wait_for_unit("prometheus-lnd-exporter.service")
wait_for_open_port(9092)
succeed("curl -sSf localhost:9092/metrics | grep '^lnd_peer_count'")
'';
};
mail = {
exporterConfig = {
enable = true;
configuration = {
monitoringInterval = "2s";
mailCheckTimeout = "10s";
servers = [{
name = "testserver";
server = "localhost";
port = 25;
from = "mail-exporter@localhost";
to = "mail-exporter@localhost";
detectionDir = "/var/spool/mail/mail-exporter/new";
}];
};
};
metricProvider = {
services.postfix.enable = true;
systemd.services.prometheus-mail-exporter = {
after = [ "postfix.service" ];
requires = [ "postfix.service" ];
serviceConfig = {
ExecStartPre = [
"${pkgs.writeShellScript "create-maildir" ''
mkdir -p -m 0700 mail-exporter/new
''}"
];
ProtectHome = true;
ReadOnlyPaths = "/";
ReadWritePaths = "/var/spool/mail";
WorkingDirectory = "/var/spool/mail";
};
};
users.users.mailexporter = {
isSystemUser = true;
group = "mailexporter";
};
users.groups.mailexporter = {};
};
exporterTest = ''
wait_for_unit("postfix.service")
wait_for_unit("prometheus-mail-exporter.service")
wait_for_open_port(9225)
wait_until_succeeds(
"curl -sSf http://localhost:9225/metrics | grep 'mail_deliver_success{configname=\"testserver\"} 1'"
)
'';
};
mikrotik = {
exporterConfig = {
enable = true;
extraFlags = [ "-timeout=1s" ];
configuration = {
devices = [
{
name = "router";
address = "192.168.42.48";
user = "prometheus";
password = "shh";
}
];
features = {
bgp = true;
dhcp = true;
dhcpl = true;
dhcpv6 = true;
health = true;
routes = true;
poe = true;
pools = true;
optics = true;
w60g = true;
wlansta = true;
wlanif = true;
monitor = true;
ipsec = true;
};
};
};
exporterTest = ''
wait_for_unit("prometheus-mikrotik-exporter.service")
wait_for_open_port(9436)
succeed(
"curl -sSf http://localhost:9436/metrics | grep 'mikrotik_scrape_collector_success{device=\"router\"} 0'"
)
'';
};
modemmanager = {
exporterConfig = {
enable = true;
refreshRate = "10s";
};
metricProvider = {
# ModemManager is installed when NetworkManager is enabled. Ensure it is
# started and is wanted by NM and the exporter to start everything up
# in the right order.
networking.networkmanager.enable = true;
systemd.services.ModemManager = {
enable = true;
wantedBy = [ "NetworkManager.service" "prometheus-modemmanager-exporter.service" ];
};
};
exporterTest = ''
wait_for_unit("ModemManager.service")
wait_for_unit("prometheus-modemmanager-exporter.service")
wait_for_open_port(9539)
succeed(
"curl -sSf http://localhost:9539/metrics | grep 'modemmanager_info'"
)
'';
};
mysqld = {
exporterConfig = {
enable = true;
runAsLocalSuperUser = true;
configFile = pkgs.writeText "test-prometheus-exporter-mysqld-config.my-cnf" ''
[client]
user = exporter
password = snakeoilpassword
'';
};
metricProvider = {
services.mysql = {
enable = true;
package = pkgs.mariadb;
initialScript = pkgs.writeText "mysql-init-script.sql" ''
CREATE USER 'exporter'@'localhost'
IDENTIFIED BY 'snakeoilpassword'
WITH MAX_USER_CONNECTIONS 3;
GRANT PROCESS, REPLICATION CLIENT, SLAVE MONITOR, SELECT ON *.* TO 'exporter'@'localhost';
'';
};
};
exporterTest = ''
wait_for_unit("prometheus-mysqld-exporter.service")
wait_for_open_port(9104)
wait_for_unit("mysql.service")
succeed("curl -sSf http://localhost:9104/metrics | grep 'mysql_up 1'")
systemctl("stop mysql.service")
succeed("curl -sSf http://localhost:9104/metrics | grep 'mysql_up 0'")
systemctl("start mysql.service")
wait_for_unit("mysql.service")
succeed("curl -sSf http://localhost:9104/metrics | grep 'mysql_up 1'")
'';
};
nextcloud = {
exporterConfig = {
enable = true;
passwordFile = "/var/nextcloud-pwfile";
url = "http://localhost";
};
metricProvider = {
systemd.services.nc-pwfile =
let
passfile = (pkgs.writeText "pwfile" "snakeoilpw");
in
{
requiredBy = [ "prometheus-nextcloud-exporter.service" ];
before = [ "prometheus-nextcloud-exporter.service" ];
serviceConfig.ExecStart = ''
${pkgs.coreutils}/bin/install -o nextcloud-exporter -m 0400 ${passfile} /var/nextcloud-pwfile
'';
};
services.nginx = {
enable = true;
virtualHosts."localhost" = {
basicAuth.nextcloud-exporter = "snakeoilpw";
locations."/" = {
root = "${pkgs.prometheus-nextcloud-exporter.src}/serverinfo/testdata";
tryFiles = "/negative-space.json =404";
};
};
};
};
exporterTest = ''
wait_for_unit("nginx.service")
wait_for_unit("prometheus-nextcloud-exporter.service")
wait_for_open_port(9205)
succeed("curl -sSf http://localhost:9205/metrics | grep 'nextcloud_up 1'")
'';
};
nginx = {
exporterConfig = {
enable = true;
constLabels = [ "foo=bar" ];
};
metricProvider = {
services.nginx = {
enable = true;
statusPage = true;
virtualHosts."test".extraConfig = "return 204;";
};
};
exporterTest = ''
wait_for_unit("nginx.service")
wait_for_unit("prometheus-nginx-exporter.service")
wait_for_open_port(9113)
succeed("curl -sSf http://localhost:9113/metrics | grep 'nginx_up{foo=\"bar\"} 1'")
'';
};
nginxlog = {
exporterConfig = {
enable = true;
group = "nginx";
settings = {
namespaces = [
{
name = "filelogger";
source = {
files = [ "/var/log/nginx/filelogger.access.log" ];
};
}
{
name = "syslogger";
source = {
syslog = {
listen_address = "udp://127.0.0.1:10000";
format = "rfc3164";
tags = [ "nginx" ];
};
};
}
];
};
};
metricProvider = {
services.nginx = {
enable = true;
httpConfig = ''
server {
listen 80;
server_name filelogger.local;
access_log /var/log/nginx/filelogger.access.log;
}
server {
listen 81;
server_name syslogger.local;
access_log syslog:server=127.0.0.1:10000,tag=nginx,severity=info;
}
'';
};
};
exporterTest = ''
wait_for_unit("nginx.service")
wait_for_unit("prometheus-nginxlog-exporter.service")
wait_for_open_port(9117)
wait_for_open_port(80)
wait_for_open_port(81)
succeed("curl http://localhost")
execute("sleep 1")
succeed(
"curl -sSf http://localhost:9117/metrics | grep 'filelogger_http_response_count_total' | grep 1"
)
succeed("curl http://localhost:81")
execute("sleep 1")
succeed(
"curl -sSf http://localhost:9117/metrics | grep 'syslogger_http_response_count_total' | grep 1"
)
'';
};
node = {
exporterConfig = {
enable = true;
};
exporterTest = ''
wait_for_unit("prometheus-node-exporter.service")
wait_for_open_port(9100)
succeed(
"curl -sSf http://localhost:9100/metrics | grep 'node_exporter_build_info{.\\+} 1'"
)
'';
};
openldap = {
exporterConfig = {
enable = true;
ldapCredentialFile = "${pkgs.writeText "exporter.yml" ''
ldapUser: "cn=root,dc=example"
ldapPass: "notapassword"
''}";
};
metricProvider = {
services.openldap = {
enable = true;
settings.children = {
"cn=schema".includes = [
"${pkgs.openldap}/etc/schema/core.ldif"
"${pkgs.openldap}/etc/schema/cosine.ldif"
"${pkgs.openldap}/etc/schema/inetorgperson.ldif"
"${pkgs.openldap}/etc/schema/nis.ldif"
];
"olcDatabase={1}mdb" = {
attrs = {
objectClass = [ "olcDatabaseConfig" "olcMdbConfig" ];
olcDatabase = "{1}mdb";
olcDbDirectory = "/var/lib/openldap/db";
olcSuffix = "dc=example";
olcRootDN = {
# cn=root,dc=example
base64 = "Y249cm9vdCxkYz1leGFtcGxl";
};
olcRootPW = {
path = "${pkgs.writeText "rootpw" "notapassword"}";
};
};
};
"olcDatabase={2}monitor".attrs = {
objectClass = [ "olcDatabaseConfig" ];
olcDatabase = "{2}monitor";
olcAccess = [ "to dn.subtree=cn=monitor by users read" ];
};
};
declarativeContents."dc=example" = ''
dn: dc=example
objectClass: domain
dc: example
dn: ou=users,dc=example
objectClass: organizationalUnit
ou: users
'';
};
};
exporterTest = ''
wait_for_unit("prometheus-openldap-exporter.service")
wait_for_open_port(389)
wait_for_open_port(9330)
wait_until_succeeds(
"curl -sSf http://localhost:9330/metrics | grep 'openldap_scrape{result=\"ok\"} 1'"
)
'';
};
pgbouncer = {
exporterConfig = {
enable = true;
connectionStringFile = pkgs.writeText "connection.conf" "postgres://admin:@localhost:6432/pgbouncer?sslmode=disable";
};
metricProvider = {
services.postgresql.enable = true;
services.pgbouncer = {
enable = true;
settings = {
pgbouncer = {
listen_addr = "*";
auth_type = "any";
max_client_conn = 99;
# https://github.com/prometheus-community/pgbouncer_exporter#pgbouncer-configuration
ignore_startup_parameters = "extra_float_digits";
};
databases = {
postgres = concatStringsSep " " [
"host=/run/postgresql"
"port=5432"
"auth_user=postgres"
"dbname=postgres"
];
};
};
};
};
exporterTest = ''
wait_for_unit("postgresql.service")
wait_for_unit("pgbouncer.service")
wait_for_unit("prometheus-pgbouncer-exporter.service")
wait_for_open_port(9127)
succeed("curl -sSf http://localhost:9127/metrics | grep 'pgbouncer_up 1'")
succeed(
"curl -sSf http://localhost:9127/metrics | grep 'pgbouncer_config_max_client_connections 99'"
)
'';
};
php-fpm = {
nodeName = "php_fpm";
exporterConfig = {
enable = true;
environmentFile = pkgs.writeTextFile {
name = "/tmp/prometheus-php-fpm-exporter.env";
text = ''
PHP_FPM_SCRAPE_URI="tcp://127.0.0.1:9000/status"
'';
};
};
metricProvider = {
users.users."php-fpm-exporter" = {
isSystemUser = true;
group = "php-fpm-exporter";
};
users.groups."php-fpm-exporter" = {};
services.phpfpm.pools."php-fpm-exporter" = {
user = "php-fpm-exporter";
group = "php-fpm-exporter";
settings = {
"pm" = "dynamic";
"pm.max_children" = 32;
"pm.max_requests" = 500;
"pm.start_servers" = 2;
"pm.min_spare_servers" = 2;
"pm.max_spare_servers" = 5;
"pm.status_path" = "/status";
"listen" = "127.0.0.1:9000";
"listen.allowed_clients" = "127.0.0.1";
};
phpEnv."PATH" = makeBinPath [ pkgs.php ];
};
};
exporterTest = ''
wait_for_unit("phpfpm-php-fpm-exporter.service")
wait_for_unit("prometheus-php-fpm-exporter.service")
succeed("curl -sSf http://localhost:9253/metrics | grep 'phpfpm_up{.*} 1'")
'';
};
ping = {
exporterConfig = {
enable = true;
settings = {
targets = [ {
"localhost" = {
alias = "local machine";
env = "prod";
type = "domain";
};
} {
"127.0.0.1" = {
alias = "local machine";
type = "v4";
};
} {
"::1" = {
alias = "local machine";
type = "v6";
};
} {
"google.com" = {};
} ];
dns = {};
ping = {
interval = "2s";
timeout = "3s";
history-size = 42;
payload-size = 56;
};
log = {
level = "warn";
};
};
};
exporterTest = ''
wait_for_unit("prometheus-ping-exporter.service")
wait_for_open_port(9427)
succeed("curl -sSf http://localhost:9427/metrics | grep 'ping_up{.*} 1'")
'';
};
postfix = {
exporterConfig = {
enable = true;
};
metricProvider = {
services.postfix.enable = true;
};
exporterTest = ''
wait_for_unit("prometheus-postfix-exporter.service")
wait_for_file("/var/lib/postfix/queue/public/showq")
wait_for_open_port(9154)
wait_until_succeeds(
"curl -sSf http://localhost:9154/metrics | grep 'postfix_up{path=\"/var/lib/postfix/queue/public/showq\"} 1'"
)
succeed(
"curl -sSf http://localhost:9154/metrics | grep 'postfix_smtpd_connects_total 0'"
)
succeed("curl -sSf http://localhost:9154/metrics | grep 'postfix_up{.*} 1'")
'';
};
postgres = {
exporterConfig = {
enable = true;
runAsLocalSuperUser = true;
};
metricProvider = {
services.postgresql.enable = true;
};
exporterTest = ''
wait_for_unit("prometheus-postgres-exporter.service")
wait_for_open_port(9187)
wait_for_unit("postgresql.service")
succeed(
"curl -sSf http://localhost:9187/metrics | grep 'pg_exporter_last_scrape_error 0'"
)
succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 1'")
systemctl("stop postgresql.service")
succeed(
"curl -sSf http://localhost:9187/metrics | grep -v 'pg_exporter_last_scrape_error 0'"
)
succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 0'")
systemctl("start postgresql.service")
wait_for_unit("postgresql.service")
succeed(
"curl -sSf http://localhost:9187/metrics | grep 'pg_exporter_last_scrape_error 0'"
)
succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 1'")
'';
};
process = {
exporterConfig = {
enable = true;
settings.process_names = [
# Remove nix store path from process name
{ name = "{{.Matches.Wrapped}} {{ .Matches.Args }}"; cmdline = [ "^/nix/store[^ ]*/(?P<Wrapped>[^ /]*) (?P<Args>.*)" ]; }
];
};
exporterTest = ''
wait_for_unit("prometheus-process-exporter.service")
wait_for_open_port(9256)
wait_until_succeeds(
"curl -sSf localhost:9256/metrics | grep -q '{}'".format(
'namedprocess_namegroup_cpu_seconds_total{groupname="process-exporter '
)
)
'';
};
pve = let
pveExporterEnvFile = pkgs.writeTextFile {
name = "pve.env";
text = ''
PVE_USER="test_user@pam"
PVE_PASSWORD="hunter3"
PVE_VERIFY_SSL="false"
'';
};
in {
exporterConfig = {
enable = true;
environmentFile = pveExporterEnvFile;
};
exporterTest = ''
wait_for_unit("prometheus-pve-exporter.service")
wait_for_open_port(9221)
wait_until_succeeds("curl localhost:9221")
'';
};
py-air-control = {
nodeName = "py_air_control";
exporterConfig = {
enable = true;
deviceHostname = "127.0.0.1";
};
exporterTest = ''
wait_for_unit("prometheus-py-air-control-exporter.service")
wait_for_open_port(9896)
succeed(
"curl -sSf http://localhost:9896/metrics | grep 'py_air_control_sampling_error_total'"
)
'';
};
redis = {
exporterConfig = {
enable = true;
};
metricProvider.services.redis.servers."".enable = true;
exporterTest = ''
wait_for_unit("redis.service")
wait_for_unit("prometheus-redis-exporter.service")
wait_for_open_port(6379)
wait_for_open_port(9121)
wait_until_succeeds("curl -sSf localhost:9121/metrics | grep 'redis_up 1'")
'';
};
restic =
let
repository = "rest:http://127.0.0.1:8000";
passwordFile = pkgs.writeText "restic-test-password" "test-password";
in
{
exporterConfig = {
enable = true;
inherit repository passwordFile;
};
metricProvider = {
services.restic.server = {
enable = true;
extraFlags = [ "--no-auth" ];
};
environment.systemPackages = [ pkgs.restic ];
};
exporterTest = ''
# prometheus-restic-exporter.service fails without initialised repository
systemctl("stop prometheus-restic-exporter.service")
# Initialise the repository
wait_for_unit("restic-rest-server.service")
wait_for_open_port(8000)
succeed("restic init --repo ${repository} --password-file ${passwordFile}")
systemctl("start prometheus-restic-exporter.service")
wait_for_unit("prometheus-restic-exporter.service")
wait_for_open_port(9753)
wait_until_succeeds("curl -sSf localhost:9753/metrics | grep 'restic_check_success 1.0'")
'';
};
rspamd = {
exporterConfig = {
enable = true;
};
metricProvider = {
services.rspamd.enable = true;
};
exporterTest = ''
wait_for_unit("rspamd.service")
wait_for_unit("prometheus-rspamd-exporter.service")
wait_for_open_port(11334)
wait_for_open_port(7980)
wait_until_succeeds(
"curl -sSf 'localhost:7980/probe?target=http://localhost:11334/stat' | grep 'rspamd_scanned{host=\"rspamd\"} 0'"
)
'';
};
rtl_433 = {
exporterConfig = {
enable = true;
};
metricProvider = {
# Mock rtl_433 binary to return a dummy metric stream.
nixpkgs.overlays = [
(self: super: {
rtl_433 = self.runCommand "rtl_433" { } ''
mkdir -p "$out/bin"
cat <<EOF > "$out/bin/rtl_433"
#!/bin/sh
while true; do
printf '{"time" : "2020-04-26 13:37:42", "model" : "zopieux", "id" : 55, "channel" : 3, "temperature_C" : 18.000}\n'
sleep 4
done
EOF
chmod +x "$out/bin/rtl_433"
'';
})
];
};
exporterTest = ''
wait_for_unit("prometheus-rtl_433-exporter.service")
wait_for_open_port(9550)
wait_until_succeeds(
"curl -sSf localhost:9550/metrics | grep '{}'".format(
'rtl_433_temperature_celsius{channel="3",id="55",location="",model="zopieux"} 18'
)
)
'';
};
sabnzbd = {
exporterConfig = {
enable = true;
servers = [{
baseUrl = "http://localhost:8080";
apiKeyFile = "/var/sabnzbd-apikey";
}];
};
metricProvider = {
services.sabnzbd.enable = true;
# unrar is required for sabnzbd
nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (pkgs.lib.getName pkg) [ "unrar" ];
# extract the generated api key before starting
systemd.services.sabnzbd-apikey = {
requires = [ "sabnzbd.service" ];
after = [ "sabnzbd.service" ];
requiredBy = [ "prometheus-sabnzbd-exporter.service" ];
before = [ "prometheus-sabnzbd-exporter.service" ];
script = ''
grep -Po '^api_key = \K.+' /var/lib/sabnzbd/sabnzbd.ini > /var/sabnzbd-apikey
'';
};
};
exporterTest = ''
wait_for_unit("sabnzbd.service")
wait_for_unit("prometheus-sabnzbd-exporter.service")
wait_for_open_port(8080)
wait_for_open_port(9387)
wait_until_succeeds(
"curl -sSf 'localhost:9387/metrics' | grep 'sabnzbd_queue_size{sabnzbd_instance=\"http://localhost:8080\"} 0.0'"
)
'';
};
scaphandre = {
exporterConfig = {
enable = true;
};
metricProvider = {
boot.kernelModules = [ "intel_rapl_common" ];
};
exporterTest = ''
wait_for_unit("prometheus-scaphandre-exporter.service")
wait_for_open_port(8080)
wait_until_succeeds(
"curl -sSf 'localhost:8080/metrics'"
)
'';
};
shelly = {
exporterConfig = {
enable = true;
metrics-file = "${pkgs.writeText "test.json" ''{}''}";
};
exporterTest = ''
wait_for_unit("prometheus-shelly-exporter.service")
wait_for_open_port(9784)
wait_until_succeeds(
"curl -sSf 'localhost:9784/metrics'"
)
'';
};
script = {
exporterConfig = {
enable = true;
settings.scripts = [
{ name = "success"; script = "sleep 1"; }
];
};
exporterTest = ''
wait_for_unit("prometheus-script-exporter.service")
wait_for_open_port(9172)
wait_until_succeeds(
"curl -sSf 'localhost:9172/probe?name=success' | grep -q '{}'".format(
'script_success{script="success"} 1'
)
)
'';
};
smartctl = {
exporterConfig = {
enable = true;
devices = [
"/dev/vda"
];
};
exporterTest = ''
wait_until_succeeds(
'journalctl -eu prometheus-smartctl-exporter.service -o cat | grep "Unable to detect device type"'
)
'';
};
smokeping = {
exporterConfig = {
enable = true;
hosts = [ "127.0.0.1" ];
};
exporterTest = ''
wait_for_unit("prometheus-smokeping-exporter.service")
wait_for_open_port(9374)
wait_until_succeeds(
"curl -sSf localhost:9374/metrics | grep '{}' | grep -v ' 0$'".format(
'smokeping_requests_total{host="127.0.0.1",ip="127.0.0.1",source=""} '
)
)
wait_until_succeeds(
"curl -sSf localhost:9374/metrics | grep '{}'".format(
'smokeping_response_ttl{host="127.0.0.1",ip="127.0.0.1",source=""}'
)
)
'';
};
snmp = {
exporterConfig = {
enable = true;
configuration = {
auths.public_v2 = {
community = "public";
version = 2;
};
};
};
exporterTest = ''
wait_for_unit("prometheus-snmp-exporter.service")
wait_for_open_port(9116)
succeed("curl -sSf localhost:9116/metrics | grep 'snmp_request_errors_total 0'")
'';
};
sql = {
exporterConfig = {
configuration.jobs.points = {
interval = "1m";
connections = [
"postgres://prometheus-sql-exporter@/data?host=/run/postgresql&sslmode=disable"
];
queries = {
points = {
labels = [ "name" ];
help = "Amount of points accumulated per person";
values = [ "amount" ];
query = "SELECT SUM(amount) as amount, name FROM points GROUP BY name";
};
};
};
enable = true;
user = "prometheus-sql-exporter";
};
metricProvider = {
services.postgresql = {
enable = true;
initialScript = builtins.toFile "init.sql" ''
CREATE DATABASE data;
\c data;
CREATE TABLE points (amount INT, name TEXT);
INSERT INTO points(amount, name) VALUES (1, 'jack');
INSERT INTO points(amount, name) VALUES (2, 'jill');
INSERT INTO points(amount, name) VALUES (3, 'jack');
CREATE USER "prometheus-sql-exporter";
GRANT ALL PRIVILEGES ON DATABASE data TO "prometheus-sql-exporter";
GRANT SELECT ON points TO "prometheus-sql-exporter";
'';
};
systemd.services.prometheus-sql-exporter.after = [ "postgresql.service" ];
};
exporterTest = ''
wait_for_unit("prometheus-sql-exporter.service")
wait_for_open_port(9237)
succeed("curl http://localhost:9237/metrics | grep -c 'sql_points{' | grep 2")
'';
};
statsd = {
exporterConfig = {
enable = true;
};
exporterTest = ''
wait_for_unit("prometheus-statsd-exporter.service")
wait_for_open_port(9102)
succeed("curl http://localhost:9102/metrics | grep 'statsd_exporter_build_info{'")
wait_until_succeeds(
"echo 'test.udp:1|c' > /dev/udp/localhost/9125 && \
curl http://localhost:9102/metrics | grep 'test_udp 1'",
timeout=10
)
wait_until_succeeds(
"echo 'test.tcp:1|c' > /dev/tcp/localhost/9125 && \
curl http://localhost:9102/metrics | grep 'test_tcp 1'",
timeout=10
)
'';
};
surfboard = {
exporterConfig = {
enable = true;
modemAddress = "localhost";
};
metricProvider = {
systemd.services.prometheus-surfboard-exporter.after = [ "nginx.service" ];
services.nginx = {
enable = true;
virtualHosts.localhost.locations."/cgi-bin/status".extraConfig = ''
return 204;
'';
};
};
exporterTest = ''
wait_for_unit("nginx.service")
wait_for_open_port(80)
wait_for_unit("prometheus-surfboard-exporter.service")
wait_for_open_port(9239)
succeed("curl -sSf localhost:9239/metrics | grep 'surfboard_up 1'")
'';
};
systemd = {
exporterConfig = {
enable = true;
extraFlags = [
"--systemd.collector.enable-restart-count"
];
};
metricProvider = { };
exporterTest = ''
wait_for_unit("prometheus-systemd-exporter.service")
wait_for_open_port(9558)
wait_until_succeeds(
"curl -sSf localhost:9558/metrics | grep '{}'".format(
'systemd_unit_state{name="basic.target",state="active",type="target"} 1'
)
)
succeed(
"curl -sSf localhost:9558/metrics | grep '{}'".format(
'systemd_service_restart_total{name="prometheus-systemd-exporter.service"} 0'
)
)
'';
};
tor = {
exporterConfig = {
enable = true;
};
metricProvider = {
# Note: this does not connect the test environment to the Tor network.
# Client, relay, bridge or exit connectivity are disabled by default.
services.tor.enable = true;
services.tor.settings.ControlPort = 9051;
};
exporterTest = ''
wait_for_unit("tor.service")
wait_for_open_port(9051)
wait_for_unit("prometheus-tor-exporter.service")
wait_for_open_port(9130)
succeed("curl -sSf localhost:9130/metrics | grep 'tor_version{.\\+} 1'")
'';
};
unpoller = {
nodeName = "unpoller";
exporterConfig.enable = true;
exporterConfig.controllers = [{ }];
exporterTest = ''
wait_until_succeeds(
'journalctl -eu prometheus-unpoller-exporter.service -o cat | grep "Connection Error"'
)
'';
};
unbound = {
exporterConfig = {
enable = true;
unbound.host = "unix:///run/unbound/unbound.ctl";
};
metricProvider = {
services.unbound = {
enable = true;
localControlSocketPath = "/run/unbound/unbound.ctl";
};
systemd.services.prometheus-unbound-exporter.serviceConfig = {
SupplementaryGroups = [ "unbound" ];
};
};
exporterTest = ''
wait_for_unit("unbound.service")
wait_for_unit("prometheus-unbound-exporter.service")
wait_for_open_port(9167)
wait_until_succeeds("curl -sSf localhost:9167/metrics | grep 'unbound_up 1'")
'';
};
v2ray = {
exporterConfig = {
enable = true;
};
metricProvider = {
systemd.services.prometheus-nginx-exporter.after = [ "v2ray.service" ];
services.v2ray = {
enable = true;
config = {
stats = {};
api = {
tag = "api";
services = [ "StatsService" ];
};
inbounds = [
{
port = 1080;
listen = "127.0.0.1";
protocol = "http";
}
{
listen = "127.0.0.1";
port = 54321;
protocol = "dokodemo-door";
settings = { address = "127.0.0.1"; };
tag = "api";
}
];
outbounds = [
{
protocol = "freedom";
}
{
protocol = "freedom";
settings = {};
tag = "api";
}
];
routing = {
strategy = "rules";
settings = {
rules = [
{
inboundTag = [ "api" ];
outboundTag = "api";
type = "field";
}
];
};
};
};
};
};
exporterTest = ''
wait_for_unit("prometheus-v2ray-exporter.service")
wait_for_open_port(9299)
succeed("curl -sSf localhost:9299/scrape | grep 'v2ray_up 1'")
'';
};
varnish = {
exporterConfig = {
enable = true;
instance = "/var/spool/varnish/varnish";
group = "varnish";
};
metricProvider = {
systemd.services.prometheus-varnish-exporter.after = [
"varnish.service"
];
services.varnish = {
enable = true;
config = ''
vcl 4.0;
backend default {
.host = "127.0.0.1";
.port = "80";
}
'';
};
};
exporterTest = ''
wait_for_unit("prometheus-varnish-exporter.service")
wait_for_open_port(6081)
wait_for_open_port(9131)
succeed("curl -sSf http://localhost:9131/metrics | grep 'varnish_up 1'")
'';
};
wireguard = let
snakeoil = import ./wireguard/snakeoil-keys.nix;
publicKeyWithoutNewlines = replaceStrings [ "\n" ] [ "" ] snakeoil.peer1.publicKey;
in
{
exporterConfig.enable = true;
metricProvider = {
networking.wireguard.interfaces.wg0 = {
ips = [ "10.23.42.1/32" "fc00::1/128" ];
listenPort = 23542;
inherit (snakeoil.peer0) privateKey;
peers = singleton {
allowedIPs = [ "10.23.42.2/32" "fc00::2/128" ];
inherit (snakeoil.peer1) publicKey;
};
};
systemd.services.prometheus-wireguard-exporter.after = [ "wireguard-wg0.service" ];
};
exporterTest = ''
wait_for_unit("prometheus-wireguard-exporter.service")
wait_for_open_port(9586)
wait_until_succeeds(
"curl -sSf http://localhost:9586/metrics | grep '${publicKeyWithoutNewlines}'"
)
'';
};
zfs = {
exporterConfig = {
enable = true;
};
metricProvider = {
boot.supportedFilesystems = [ "zfs" ];
networking.hostId = "7327ded7";
};
exporterTest = ''
wait_for_unit("prometheus-zfs-exporter.service")
wait_for_unit("zfs.target")
wait_for_open_port(9134)
wait_until_succeeds("curl -f localhost:9134/metrics | grep 'zfs_scrape_collector_success{.*} 1'")
'';
};
};
in
mapAttrs
(exporter: testConfig: (makeTest (
let
nodeName = testConfig.nodeName or exporter;
in
{
name = "prometheus-${exporter}-exporter";
nodes.${nodeName} = mkMerge [{
services.prometheus.exporters.${exporter} = testConfig.exporterConfig;
} testConfig.metricProvider or { }];
testScript = ''
${nodeName}.start()
${concatStringsSep "\n" (map (line:
if builtins.any (b: b) [
(builtins.match "^[[:space:]]*$" line != null)
(builtins.substring 0 1 line == "#")
(builtins.substring 0 1 line == " ")
(builtins.substring 0 1 line == ")")
]
then line
else "${nodeName}.${line}"
) (splitString "\n" (removeSuffix "\n" testConfig.exporterTest)))}
${nodeName}.shutdown()
'';
meta = with maintainers; {
maintainers = [ willibutz ];
};
}
)))
exporterTests