diff --git a/nixos/modules/services/logging/journalbeat.nix b/nixos/modules/services/logging/journalbeat.nix index 8186a3b02c37..89f53b1b2454 100644 --- a/nixos/modules/services/logging/journalbeat.nix +++ b/nixos/modules/services/logging/journalbeat.nix @@ -5,11 +5,13 @@ with lib; let cfg = config.services.journalbeat; + lt6 = builtins.compareVersions cfg.package.version "6" < 0; + journalbeatYml = pkgs.writeText "journalbeat.yml" '' name: ${cfg.name} tags: ${builtins.toJSON cfg.tags} - journalbeat.cursor_state_file: ${cfg.stateDir}/cursor-state + ${optionalString lt6 "journalbeat.cursor_state_file: /var/lib/${cfg.stateDir}/cursor-state"} ${cfg.extraConfig} ''; @@ -22,6 +24,16 @@ in enable = mkEnableOption "journalbeat"; + package = mkOption { + type = types.package; + default = pkgs.journalbeat; + defaultText = "pkgs.journalbeat"; + example = literalExample "pkgs.journalbeat7"; + description = '' + The journalbeat package to use + ''; + }; + name = mkOption { type = types.str; default = "journalbeat"; @@ -36,13 +48,17 @@ in stateDir = mkOption { type = types.str; - default = "/var/lib/journalbeat"; - description = "The state directory. Journalbeat's own logs and other data are stored here."; + default = "journalbeat"; + description = '' + Directory below /var/lib/ to store journalbeat's + own logs and other data. This directory will be created automatically + using systemd's StateDirectory mechanism. + ''; }; extraConfig = mkOption { type = types.lines; - default = '' + default = optionalString lt6 '' journalbeat: seek_position: cursor cursor_seek_fallback: tail @@ -61,7 +77,16 @@ in config = mkIf cfg.enable { - systemd.services.journalbeat = with pkgs; { + assertions = [ + { + assertion = !hasPrefix "/" cfg.stateDir; + message = + "The option services.journalbeat.stateDir shouldn't be an absolute directory." + + " It should be a directory relative to /var/lib/."; + } + ]; + + systemd.services.journalbeat = { description = "Journalbeat log shipper"; wantedBy = [ "multi-user.target" ]; preStart = '' @@ -69,7 +94,13 @@ in mkdir -p ${cfg.stateDir}/logs ''; serviceConfig = { - ExecStart = "${pkgs.journalbeat}/bin/journalbeat -c ${journalbeatYml} -path.data ${cfg.stateDir}/data -path.logs ${cfg.stateDir}/logs"; + StateDirectory = cfg.stateDir; + ExecStart = '' + ${cfg.package}/bin/journalbeat \ + -c ${journalbeatYml} \ + -path.data /var/lib/${cfg.stateDir}/data \ + -path.logs /var/lib/${cfg.stateDir}/logs''; + Restart = "always"; }; }; }; diff --git a/nixos/tests/elk.nix b/nixos/tests/elk.nix index 3b3fbd73dd5f..95371ef44436 100644 --- a/nixos/tests/elk.nix +++ b/nixos/tests/elk.nix @@ -12,6 +12,11 @@ with pkgs.lib; let esUrl = "http://localhost:9200"; + totalHits = message : + "curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' " + + ''-d '{\"query\" : { \"match\" : { \"message\" : \"${message}\"}}}' '' + + "| jq .hits.total"; + mkElkTest = name : elk : let elasticsearchGe7 = builtins.compareVersions elk.elasticsearch.version "7" >= 0; in makeTest { @@ -21,7 +26,7 @@ let }; nodes = { one = - { pkgs, ... }: { + { pkgs, lib, ... }: { # Not giving the machine at least 2060MB results in elasticsearch failing with the following error: # # OpenJDK 64-Bit Server VM warning: @@ -40,6 +45,26 @@ let environment.systemPackages = [ pkgs.jq ]; services = { + + journalbeat = let lt6 = builtins.compareVersions + elk.journalbeat.version "6" < 0; in { + enable = true; + package = elk.journalbeat; + extraConfig = mkOptionDefault ('' + logging: + to_syslog: true + level: warning + metrics.enabled: false + output.elasticsearch: + hosts: [ "127.0.0.1:9200" ] + ${optionalString lt6 "template.enabled: false"} + '' + optionalString (!lt6) '' + journalbeat.inputs: + - paths: [] + seek: cursor + ''); + }; + logstash = { enable = true; package = elk.logstash; @@ -107,14 +132,19 @@ let testScript = '' startAll; + # Wait until elasticsearch is listening for connections. $one->waitForUnit("elasticsearch.service"); + $one->waitForOpenPort(9200); # Continue as long as the status is not "red". The status is probably # "yellow" instead of "green" because we are using a single elasticsearch # node which elasticsearch considers risky. # - # TODO: extend this test with multiple elasticsearch nodes and see if the status turns "green". - $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red"); + # TODO: extend this test with multiple elasticsearch nodes + # and see if the status turns "green". + $one->waitUntilSucceeds( + "curl --silent --show-error '${esUrl}/_cluster/health' " . + "| jq .status | grep -v red"); # Perform some simple logstash tests. $one->waitForUnit("logstash.service"); @@ -123,16 +153,28 @@ let # See if kibana is healthy. $one->waitForUnit("kibana.service"); - $one->waitUntilSucceeds("curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green"); + $one->waitUntilSucceeds( + "curl --silent --show-error 'http://localhost:5601/api/status' " . + "| jq .status.overall.state | grep green"); # See if logstash messages arive in elasticsearch. - $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0"); - $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0"); + $one->waitUntilSucceeds("${totalHits "flowers"} | grep -v 0"); + $one->waitUntilSucceeds("${totalHits "dragons"} | grep 0"); + + # Test if a message logged to the journal + # is ingested by elasticsearch via journalbeat. + $one->waitForUnit("journalbeat.service"); + $one->execute("echo 'Supercalifragilisticexpialidocious' | systemd-cat"); + $one->waitUntilSucceeds( + "${totalHits "Supercalifragilisticexpialidocious"} | grep -v 0"); + '' + optionalString (!elasticsearchGe7) '' # Test elasticsearch-curator. $one->systemctl("stop logstash"); $one->systemctl("start elasticsearch-curator"); - $one->waitUntilSucceeds("! curl --silent --show-error '${esUrl}/_cat/indices' | grep logstash | grep -q ^$1"); + $one->waitUntilSucceeds( + "! curl --silent --show-error '${esUrl}/_cat/indices' " . + "| grep logstash | grep -q ^$1"); ''; }; in mapAttrs mkElkTest { @@ -140,6 +182,7 @@ in mapAttrs mkElkTest { elasticsearch = pkgs.elasticsearch5; logstash = pkgs.logstash5; kibana = pkgs.kibana5; + journalbeat = pkgs.journalbeat5; }; "ELK-6" = if enableUnfree @@ -147,11 +190,13 @@ in mapAttrs mkElkTest { elasticsearch = pkgs.elasticsearch6; logstash = pkgs.logstash6; kibana = pkgs.kibana6; + journalbeat = pkgs.journalbeat6; } else { elasticsearch = pkgs.elasticsearch6-oss; logstash = pkgs.logstash6-oss; kibana = pkgs.kibana6-oss; + journalbeat = pkgs.journalbeat6; }; "ELK-7" = if enableUnfree @@ -159,10 +204,12 @@ in mapAttrs mkElkTest { elasticsearch = pkgs.elasticsearch7; logstash = pkgs.logstash7; kibana = pkgs.kibana7; + journalbeat = pkgs.journalbeat7; } else { elasticsearch = pkgs.elasticsearch7-oss; logstash = pkgs.logstash7-oss; kibana = pkgs.kibana7-oss; + journalbeat = pkgs.journalbeat7; }; } diff --git a/pkgs/misc/logging/beats/6.x.nix b/pkgs/misc/logging/beats/6.x.nix index 2cfae05c9242..5351b253eedc 100644 --- a/pkgs/misc/logging/beats/6.x.nix +++ b/pkgs/misc/logging/beats/6.x.nix @@ -45,5 +45,8 @@ in { journal entries from Linuxes with systemd. ''; buildInputs = [ systemd.dev ]; + postFixup = let libPath = stdenv.lib.makeLibraryPath [ systemd.lib ]; in '' + patchelf --set-rpath ${libPath} "$bin/bin/journalbeat" + ''; }; } diff --git a/pkgs/misc/logging/beats/7.x.nix b/pkgs/misc/logging/beats/7.x.nix index 629f4128acb7..12e08dfeec69 100644 --- a/pkgs/misc/logging/beats/7.x.nix +++ b/pkgs/misc/logging/beats/7.x.nix @@ -45,5 +45,8 @@ in { journal entries from Linuxes with systemd. ''; buildInputs = [ systemd.dev ]; + postFixup = let libPath = stdenv.lib.makeLibraryPath [ systemd.lib ]; in '' + patchelf --set-rpath ${libPath} "$bin/bin/journalbeat" + ''; }; } diff --git a/pkgs/tools/system/journalbeat/default.nix b/pkgs/tools/system/journalbeat/default.nix index 20951fe52406..0f13d2d3da1c 100644 --- a/pkgs/tools/system/journalbeat/default.nix +++ b/pkgs/tools/system/journalbeat/default.nix @@ -1,24 +1,9 @@ -{ lib, pkgs, buildGoPackage, fetchFromGitHub, makeWrapper }: - -let - - libPath = lib.makeLibraryPath [ pkgs.systemd.lib ]; - -in buildGoPackage rec { +{ lib, systemd, buildGoPackage, fetchFromGitHub, makeWrapper }: +buildGoPackage rec { name = "journalbeat-${version}"; version = "5.6.8"; - goPackagePath = "github.com/mheese/journalbeat"; - - buildInputs = [ makeWrapper pkgs.systemd ]; - - postInstall = '' - wrapProgram $bin/bin/journalbeat \ - --argv0 journalbeat \ - --prefix LD_LIBRARY_PATH : ${libPath} - ''; - src = fetchFromGitHub { owner = "mheese"; repo = "journalbeat"; @@ -26,6 +11,14 @@ in buildGoPackage rec { sha256 = "1vgpwnwqjc93nvdpcd52748bwl3r371jb55l17bsgdzrmlcyfm8a"; }; + goPackagePath = "github.com/mheese/journalbeat"; + + buildInputs = [ systemd.dev ]; + + postFixup = let libPath = lib.makeLibraryPath [ systemd.lib ]; in '' + patchelf --set-rpath ${libPath} "$bin/bin/journalbeat" + ''; + meta = with lib; { homepage = https://github.com/mheese/journalbeat; description = "Journalbeat is a log shipper from systemd/journald to Logstash/Elasticsearch"; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 7ca2dda5abc9..69c25f0de99c 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -976,6 +976,8 @@ in metricbeat5 packetbeat5; + journalbeat5 = callPackage ../tools/system/journalbeat { }; + inherit (callPackages ../misc/logging/beats/6.x.nix { # XXX: this is failing with Go 1.12. Error is related to cgo, an # update to this package might fix it. @@ -1002,6 +1004,7 @@ in heartbeat = heartbeat6; metricbeat = metricbeat6; packetbeat = packetbeat6; + journalbeat = journalbeat6; bfr = callPackage ../tools/misc/bfr { }; @@ -3787,8 +3790,6 @@ in joplin-desktop = callPackage ../applications/misc/joplin-desktop { }; - journalbeat = callPackage ../tools/system/journalbeat { }; - journaldriver = callPackage ../tools/misc/journaldriver { }; jp = callPackage ../development/tools/jp { };