diff --git a/nixos/modules/services/monitoring/dd-agent/dd-agent-defaults.nix b/nixos/modules/services/monitoring/dd-agent/dd-agent-defaults.nix new file mode 100644 index 000000000000..045128197421 --- /dev/null +++ b/nixos/modules/services/monitoring/dd-agent/dd-agent-defaults.nix @@ -0,0 +1,8 @@ +# Generated using update-dd-agent-default, please re-run after updating dd-agent. DO NOT EDIT MANUALLY. +[ + "auto_conf" + "agent_metrics.yaml.default" + "disk.yaml.default" + "network.yaml.default" + "ntp.yaml.default" +] diff --git a/nixos/modules/services/monitoring/dd-agent/dd-agent.nix b/nixos/modules/services/monitoring/dd-agent/dd-agent.nix index e55faed8e078..8d3d4f2065ec 100644 --- a/nixos/modules/services/monitoring/dd-agent/dd-agent.nix +++ b/nixos/modules/services/monitoring/dd-agent/dd-agent.nix @@ -16,24 +16,100 @@ let forwarder_log_file: /var/log/datadog/forwarder.log dogstatsd_log_file: /var/log/datadog/dogstatsd.log pup_log_file: /var/log/datadog/pup.log + + # proxy_host: my-proxy.com + # proxy_port: 3128 + # proxy_user: user + # proxy_password: password + + # tags: mytag0, mytag1 ${optionalString (cfg.tags != null ) "tags: ${concatStringsSep "," cfg.tags }"} - ${cfg.extraDdConfig} + + # collect_ec2_tags: no + # recent_point_threshold: 30 + # use_mount: no + # listen_port: 17123 + # graphite_listen_port: 17124 + # non_local_traffic: no + # use_curl_http_client: False + # bind_host: localhost + + # use_pup: no + # pup_port: 17125 + # pup_interface: localhost + # pup_url: http://localhost:17125 + + # dogstatsd_port : 8125 + # dogstatsd_interval : 10 + # dogstatsd_normalize : yes + # statsd_forward_host: address_of_own_statsd_server + # statsd_forward_port: 8125 + + # device_blacklist_re: .*\/dev\/mapper\/lxc-box.* + + # ganglia_host: localhost + # ganglia_port: 8651 ''; - etcfiles = - map (i: { source = if builtins.hasAttr "config" i - then pkgs.writeText "${i.name}.yaml" i.config - else "${pkgs.dd-agent}/agent/conf.d-system/${i.name}.yaml"; - target = "dd-agent/conf.d/${i.name}.yaml"; - } - ) cfg.integrations ++ - [ { source = ddConf; - target = "dd-agent/datadog.conf"; - } - ]; + diskConfig = pkgs.writeText "disk.yaml" '' + init_config: - # restart triggers - etcSources = map (i: i.source) etcfiles; + instances: + - use_mount: no + ''; + + networkConfig = pkgs.writeText "network.yaml" '' + init_config: + + instances: + # Network check only supports one configured instance + - collect_connection_state: false + excluded_interfaces: + - lo + - lo0 + ''; + + postgresqlConfig = pkgs.writeText "postgres.yaml" cfg.postgresqlConfig; + nginxConfig = pkgs.writeText "nginx.yaml" cfg.nginxConfig; + mongoConfig = pkgs.writeText "mongo.yaml" cfg.mongoConfig; + jmxConfig = pkgs.writeText "jmx.yaml" cfg.jmxConfig; + processConfig = pkgs.writeText "process.yaml" cfg.processConfig; + + etcfiles = + let + defaultConfd = import ./dd-agent-defaults.nix; + in (map (f: { source = "${pkgs.dd-agent}/agent/conf.d-system/${f}"; + target = "dd-agent/conf.d/${f}"; + }) defaultConfd) ++ [ + { source = ddConf; + target = "dd-agent/datadog.conf"; + } + { source = diskConfig; + target = "dd-agent/conf.d/disk.yaml"; + } + { source = networkConfig; + target = "dd-agent/conf.d/network.yaml"; + } ] ++ + (optional (cfg.postgresqlConfig != null) + { source = postgresqlConfig; + target = "dd-agent/conf.d/postgres.yaml"; + }) ++ + (optional (cfg.nginxConfig != null) + { source = nginxConfig; + target = "dd-agent/conf.d/nginx.yaml"; + }) ++ + (optional (cfg.mongoConfig != null) + { source = mongoConfig; + target = "dd-agent/conf.d/mongo.yaml"; + }) ++ + (optional (cfg.processConfig != null) + { source = processConfig; + target = "dd-agent/conf.d/process.yaml"; + }) ++ + (optional (cfg.jmxConfig != null) + { source = jmxConfig; + target = "dd-agent/conf.d/jmx.yaml"; + }); in { options.services.dd-agent = { @@ -63,46 +139,44 @@ in { type = types.uniq (types.nullOr types.string); }; - agent = mkOption { - description = "The dd-agent package to use. Useful when overriding the package."; - default = pkgs.dd-agent; - type = types.package; + postgresqlConfig = mkOption { + description = "Datadog PostgreSQL integration configuration"; + default = null; + type = types.uniq (types.nullOr types.string); }; - integrations = mkOption { + nginxConfig = mkOption { + description = "Datadog nginx integration configuration"; + default = null; + type = types.uniq (types.nullOr types.string); + }; + + mongoConfig = mkOption { + description = "MongoDB integration configuration"; + default = null; + type = types.uniq (types.nullOr types.string); + }; + + jmxConfig = mkOption { + description = "JMX integration configuration"; + default = null; + type = types.uniq (types.nullOr types.string); + }; + + processConfig = mkOption { description = '' - Any integrations to use. Default config used if none - specified. It is currently up to the user to make sure that - the dd-agent package used has all the dependencies chosen - integrations require in scope. - ''; - type = types.listOf (types.attrsOf types.string); - default = []; - example = '' - [ { name = "elastic"; - config = ''' - init_config: - - instances: - - url: http://localhost:9200 - '''; - } - { name = "nginx"; } - { name = "ntp"; } - { name = "network"; } - ] + Process integration configuration + + See http://docs.datadoghq.com/integrations/process/ ''; + default = null; + type = types.uniq (types.nullOr types.string); }; - extraDdConfig = mkOption { - description = "Extra settings to append to datadog agent config."; - default = ""; - type = types.string; - }; }; config = mkIf cfg.enable { - environment.systemPackages = [ cfg.agent pkgs.sysstat pkgs.procps ]; + environment.systemPackages = [ pkgs."dd-agent" pkgs.sysstat pkgs.procps ]; users.extraUsers.datadog = { description = "Datadog Agent User"; @@ -116,7 +190,7 @@ in { systemd.services.dd-agent = { description = "Datadog agent monitor"; - path = [ cfg.agent pkgs.python pkgs.sysstat pkgs.procps ]; + path = [ pkgs."dd-agent" pkgs.python pkgs.sysstat pkgs.procps ]; wantedBy = [ "multi-user.target" ]; serviceConfig = { ExecStart = "${pkgs.dd-agent}/bin/dd-agent foreground"; @@ -125,12 +199,28 @@ in { Restart = "always"; RestartSec = 2; }; - restartTriggers = [ pkgs.dd-agent ddConf ] ++ etcSources; + restartTriggers = [ pkgs.dd-agent ddConf diskConfig networkConfig postgresqlConfig nginxConfig mongoConfig jmxConfig processConfig ]; }; - systemd.services.dd-jmxfetch = lib.mkIf (builtins.any (i: i.name == "jmx") cfg.integrations) { + systemd.services.dogstatsd = { + description = "Datadog statsd"; + path = [ pkgs."dd-agent" pkgs.python pkgs.procps ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + ExecStart = "${pkgs.dd-agent}/bin/dogstatsd start"; + User = "datadog"; + Group = "datadog"; + Type = "forking"; + PIDFile = "/tmp/dogstatsd.pid"; + Restart = "always"; + RestartSec = 2; + }; + restartTriggers = [ pkgs.dd-agent ddConf diskConfig networkConfig postgresqlConfig nginxConfig mongoConfig jmxConfig processConfig ]; + }; + + systemd.services.dd-jmxfetch = lib.mkIf (cfg.jmxConfig != null) { description = "Datadog JMX Fetcher"; - path = [ cfg.agent pkgs.python pkgs.sysstat pkgs.procps pkgs.jdk ]; + path = [ pkgs."dd-agent" pkgs.python pkgs.sysstat pkgs.procps pkgs.jdk ]; wantedBy = [ "multi-user.target" ]; serviceConfig = { ExecStart = "${pkgs.dd-agent}/bin/dd-jmxfetch"; @@ -139,7 +229,7 @@ in { Restart = "always"; RestartSec = 2; }; - restartTriggers = [ cfg.agent ddConf ] ++ etcSources; + restartTriggers = [ pkgs.dd-agent ddConf diskConfig networkConfig postgresqlConfig nginxConfig mongoConfig jmxConfig ]; }; environment.etc = etcfiles; diff --git a/nixos/modules/services/monitoring/dd-agent/update-dd-agent-defaults b/nixos/modules/services/monitoring/dd-agent/update-dd-agent-defaults new file mode 100755 index 000000000000..76724173171a --- /dev/null +++ b/nixos/modules/services/monitoring/dd-agent/update-dd-agent-defaults @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +dd=$(nix-build --no-out-link -A dd-agent ../../../..) +echo '# Generated using update-dd-agent-default, please re-run after updating dd-agent. DO NOT EDIT MANUALLY.' > dd-agent-defaults.nix +echo '[' >> dd-agent-defaults.nix +echo ' "auto_conf"' >> dd-agent-defaults.nix +for f in $(find $dd/agent/conf.d-system -maxdepth 1 -type f | grep -v '\.example' | sort); do + echo " \"$(basename $f)\"" >> dd-agent-defaults.nix +done +echo ']' >> dd-agent-defaults.nix diff --git a/pkgs/tools/networking/dd-agent/default.nix b/pkgs/tools/networking/dd-agent/default.nix index 2556287570fd..d1171a3c8355 100644 --- a/pkgs/tools/networking/dd-agent/default.nix +++ b/pkgs/tools/networking/dd-agent/default.nix @@ -1,11 +1,5 @@ { stdenv, fetchFromGitHub, pythonPackages -, sysstat, unzip, makeWrapper -# We need extraBuildInputs as we want to be able to override this -# package with python packages _and_ have the produced binaries -# wrapper with their PYTHONPATH. This means overrideAttrs is not -# strong enough (it overrides too late), we need to call it -# beforehand. -, extraBuildInputs ? [ pythonPackages.psutil ] }: +, sysstat, unzip, makeWrapper }: let inherit (pythonPackages) python; docker_1_10 = pythonPackages.buildPythonPackage rec { @@ -32,42 +26,33 @@ let # due to flake8 doCheck = false; }; - version = "5.13.2"; - - integrations = fetchFromGitHub { - owner = "datadog"; - repo = "integrations-core"; - rev = version; - sha256 = "1nbjmkq0wdfndmx0qap69h2rkwkkb0632j87h9d3j99bykyav3y3"; - }; in stdenv.mkDerivation rec { + version = "5.11.2"; name = "dd-agent-${version}"; src = fetchFromGitHub { owner = "datadog"; repo = "dd-agent"; rev = version; - sha256 = "0x2bxi70l2yf0wi232qksvcscjdpjg8l7dmgg1286vqryyfazfjb"; + sha256 = "1iqxvgpsqibqw3vk79158l2pnb6y4pjhjp2d6724lm5rpz4825lx"; }; buildInputs = [ python unzip makeWrapper - pythonPackages.boto - docker_1_10 - pythonPackages.kazoo - pythonPackages.ntplib - pythonPackages.consul - pythonPackages.python-etcd - pythonPackages.pyyaml pythonPackages.requests + pythonPackages.psycopg2 + pythonPackages.psutil + pythonPackages.ntplib pythonPackages.simplejson - pythonPackages.supervisor - pythonPackages.tornado - pythonPackages.uptime - ] ++ extraBuildInputs; + pythonPackages.pyyaml + pythonPackages.pymongo_2_9_1 + pythonPackages.python-etcd + pythonPackages.consul + docker_1_10 + ]; propagatedBuildInputs = with pythonPackages; [ python tornado ]; buildCommand = '' @@ -82,24 +67,6 @@ in stdenv.mkDerivation rec { # Move out default conf.d so that /etc/dd-agent/conf.d is used mv $out/agent/conf.d $out/agent/conf.d-system - # Sometime between 5.11.2 and 5.13.2 datadog moved out all its - # checks into separate repository. Copy them back in so dd-agent - # service can easily pick and choose by copying out configs into - # its etc files. - mkdir -p $out/agent/checks.d - for i in ${toString integrations}/* # */ - do - if [ -f "$i/check.py" ]; then - if [ -f "$i/conf.yaml.default" -o -f "$i/conf.yaml.example" ]; then - local name=$(basename $i) - cp $i/check.py $out/agent/checks.d/$name.py - # Copy .default file first unless it doesn't exist then copy .default - cp $i/conf.yaml.default $out/agent/conf.d-system/$name.yaml &> /dev/null || \ - cp $i/conf.yaml.example $out/agent/conf.d-system/$name.yaml - fi - fi - done - cat > $out/bin/dd-jmxfetch <