From a2aa01be0c4536985701984c8db2bd75c3a25934 Mon Sep 17 00:00:00 2001
From: Daniel Schaefer <git@danielschaefer.me>
Date: Sun, 7 Apr 2019 05:02:32 +0200
Subject: [PATCH 01/18] nixos/cassandra: Enable CQL server by default

Resolves #50954
---
 nixos/modules/services/databases/cassandra.nix | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/nixos/modules/services/databases/cassandra.nix b/nixos/modules/services/databases/cassandra.nix
index 688938868020..af0c29405c8f 100644
--- a/nixos/modules/services/databases/cassandra.nix
+++ b/nixos/modules/services/databases/cassandra.nix
@@ -8,6 +8,7 @@ let
   cassandraConfig = flip recursiveUpdate cfg.extraConfig
     ({ commitlog_sync = "batch";
        commitlog_sync_batch_window_in_ms = 2;
+       start_native_transport = cfg.allowClients;
        partitioner = "org.apache.cassandra.dht.Murmur3Partitioner";
        endpoint_snitch = "SimpleSnitch";
        seed_provider =
@@ -162,6 +163,18 @@ in {
         XML logback configuration for cassandra
       '';
     };
+    allowClients = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Enables or disables the native transport server (CQL binary protocol).
+        This server uses the same address as the <literal>rpcAddress</literal>,
+        but the port it uses is not <literal>rpc_port</literal> but
+        <literal>native_transport_port</literal>. See the official Cassandra
+        docs for more information on these variables and set them using
+        <literal>extraConfig</literal>.
+      '';
+    };
     extraConfig = mkOption {
       type = types.attrs;
       default = {};

From 746b82bd4a6226f40b89d97856576fcaa117cb5c Mon Sep 17 00:00:00 2001
From: Daniel Schaefer <git@danielschaefer.me>
Date: Sun, 7 Apr 2019 05:02:51 +0200
Subject: [PATCH 02/18] nixos/cassandra: Allow setting of seed addresses

Allow for more intuitive specifying of seed node addresses with Nix
syntax.
---
 nixos/modules/services/databases/cassandra.nix | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/nixos/modules/services/databases/cassandra.nix b/nixos/modules/services/databases/cassandra.nix
index af0c29405c8f..ee6c092f0a45 100644
--- a/nixos/modules/services/databases/cassandra.nix
+++ b/nixos/modules/services/databases/cassandra.nix
@@ -13,7 +13,7 @@ let
        endpoint_snitch = "SimpleSnitch";
        seed_provider =
          [{ class_name = "org.apache.cassandra.locator.SimpleSeedProvider";
-            parameters = [ { seeds = "127.0.0.1"; } ];
+            parameters = [ { seeds = concatStringsSep "," cfg.seedAddresses; } ];
          }];
        data_file_directories = [ "${cfg.homeDir}/data" ];
        commitlog_directory = "${cfg.homeDir}/commitlog";
@@ -163,6 +163,16 @@ in {
         XML logback configuration for cassandra
       '';
     };
+    seedAddresses = mkOption {
+      type = types.listOf types.str;
+      default = [ "127.0.0.1" ];
+      description = ''
+        The addresses of hosts designated as contact points in the cluster. A
+        joining node contacts one of the nodes in the seeds list to learn the
+        topology of the ring.
+        Set to 127.0.0.1 for a single node cluster.
+      '';
+    };
     allowClients = mkOption {
       type = types.bool;
       default = true;

From f0031432ce0c350f8e44072471f6838ee24a21a8 Mon Sep 17 00:00:00 2001
From: Daniel Schaefer <git@danielschaefer.me>
Date: Sun, 7 Apr 2019 06:30:26 +0200
Subject: [PATCH 03/18] nixos/cassandra: Add nixos conf for Java env

---
 .../modules/services/databases/cassandra.nix  | 93 +++++++++++++++++--
 1 file changed, 86 insertions(+), 7 deletions(-)

diff --git a/nixos/modules/services/databases/cassandra.nix b/nixos/modules/services/databases/cassandra.nix
index ee6c092f0a45..f51bf824a5e2 100644
--- a/nixos/modules/services/databases/cassandra.nix
+++ b/nixos/modules/services/databases/cassandra.nix
@@ -40,8 +40,11 @@ let
         mkdir -p "$out"
 
         echo "$cassandraYaml" > "$out/cassandra.yaml"
-        ln -s "$cassandraEnvPkg" "$out/cassandra-env.sh"
         ln -s "$cassandraLogbackConfig" "$out/logback.xml"
+
+        cp "$cassandraEnvPkg" "$out/cassandra-env.sh"
+        # Delete default JMX Port, otherwise we can't set it using env variable
+        sed -i '/JMX_PORT="7199"/d' "$out/cassandra-env.sh"
       '';
     };
 in {
@@ -237,20 +240,91 @@ in {
           Options passed through to the incremental repair command.
         '';
     };
+    maxHeapSize = mkOption {
+      type = types.nullOr types.string;
+      default = null;
+      example = "4G";
+      description = ''
+        Must be left blank or set together with heapNewSize.
+        If left blank a sensible value for the available amount of RAM and CPU
+        cores is calculated.
+
+        Override to set the amount of memory to allocate to the JVM at
+        start-up. For production use you may wish to adjust this for your
+        environment. MAX_HEAP_SIZE is the total amount of memory dedicated
+        to the Java heap. HEAP_NEWSIZE refers to the size of the young
+        generation.
+
+        The main trade-off for the young generation is that the larger it
+        is, the longer GC pause times will be. The shorter it is, the more
+        expensive GC will be (usually).
+      '';
+    };
+    heapNewSize = mkOption {
+      type = types.nullOr types.string;
+      default = null;
+      example = "800M";
+      description = ''
+        Must be left blank or set together with heapNewSize.
+        If left blank a sensible value for the available amount of RAM and CPU
+        cores is calculated.
+
+        Override to set the amount of memory to allocate to the JVM at
+        start-up. For production use you may wish to adjust this for your
+        environment. HEAP_NEWSIZE refers to the size of the young
+        generation.
+
+        The main trade-off for the young generation is that the larger it
+        is, the longer GC pause times will be. The shorter it is, the more
+        expensive GC will be (usually).
+
+        The example HEAP_NEWSIZE assumes a modern 8-core+ machine for decent pause
+        times. If in doubt, and if you do not particularly want to tweak, go with
+        100 MB per physical CPU core.
+      '';
+    };
+    mallocArenaMax = mkOption {
+      type = types.nullOr types.int;
+      default = null;
+      example = 4;
+      description = ''
+        Set this to control the amount of arenas per-thread in glibc.
+      '';
+    };
+    remoteJmx = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Cassandra ships with JMX accessible *only* from localhost.
+        To enable remote JMX connections set to true.
+
+        Be sure to also enable authentication and/or TLS.
+        See: https://wiki.apache.org/cassandra/JmxSecurity
+      '';
+    };
+    jmxPort = mkOption {
+      type = types.int;
+      default = 7199;
+      description = ''
+        Specifies the default port over which Cassandra will be available for
+        JMX connections.
+        For security reasons, you should not expose this port to the internet.
+        Firewall it if needed.
+      '';
+    };
   };
 
   config = mkIf cfg.enable {
     assertions =
-      [ { assertion =
-          (cfg.listenAddress == null || cfg.listenInterface == null)
-          && !(cfg.listenAddress == null && cfg.listenInterface == null);
+      [ { assertion = (cfg.listenAddress == null) != (cfg.listenInterface == null);
           message = "You have to set either listenAddress or listenInterface";
         }
-        { assertion =
-          (cfg.rpcAddress == null || cfg.rpcInterface == null)
-          && !(cfg.rpcAddress == null && cfg.rpcInterface == null);
+        { assertion = (cfg.rpcAddress == null) != (cfg.rpcInterface == null);
           message = "You have to set either rpcAddress or rpcInterface";
         }
+        { assertion = (cfg.maxHeapSize == null) == (cfg.heapNewSize == null);
+          message = "If you set either of maxHeapSize or heapNewSize you have to set both";
+        }
       ];
     users = mkIf (cfg.user == defaultUser) {
       extraUsers."${defaultUser}" =
@@ -269,6 +343,11 @@ in {
         environment =
           { CASSANDRA_CONF = "${cassandraEtc}";
             JVM_OPTS = builtins.concatStringsSep " " cfg.jvmOpts;
+            MAX_HEAP_SIZE = toString cfg.maxHeapSize;
+            HEAP_NEWSIZE = toString cfg.heapNewSize;
+            MALLOC_ARENA_MAX = toString cfg.mallocArenaMax;
+            LOCAL_JMX = if cfg.remoteJmx then "no" else "yes";
+            JMX_PORT = toString cfg.jmxPort;
           };
         wantedBy = [ "multi-user.target" ];
         serviceConfig =

From c1991fb18d2a58eefeba0a49bb683eb4e561263f Mon Sep 17 00:00:00 2001
From: Daniel Schaefer <git@danielschaefer.me>
Date: Sun, 7 Apr 2019 07:22:41 +0200
Subject: [PATCH 04/18] nixos/cassandra: Add clusterName

---
 nixos/modules/services/databases/cassandra.nix | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/nixos/modules/services/databases/cassandra.nix b/nixos/modules/services/databases/cassandra.nix
index f51bf824a5e2..3c5a47682686 100644
--- a/nixos/modules/services/databases/cassandra.nix
+++ b/nixos/modules/services/databases/cassandra.nix
@@ -9,6 +9,7 @@ let
     ({ commitlog_sync = "batch";
        commitlog_sync_batch_window_in_ms = 2;
        start_native_transport = cfg.allowClients;
+       cluster_name = cfg.clusterName;
        partitioner = "org.apache.cassandra.dht.Murmur3Partitioner";
        endpoint_snitch = "SimpleSnitch";
        seed_provider =
@@ -52,6 +53,15 @@ in {
     enable = mkEnableOption ''
       Apache Cassandra – Scalable and highly available database.
     '';
+    clusterName = mkOption {
+      type = types.str;
+      default = "NixOS Test Cluster";
+      description = ''
+        The name of the cluster.
+        This setting prevents nodes in one logical cluster from joining
+        another. All nodes in a cluster must have the same value.
+      '';
+    };
     user = mkOption {
       type = types.str;
       default = defaultUser;

From 6778ee186299ab56f141785a88abfb76f6814147 Mon Sep 17 00:00:00 2001
From: Daniel Schaefer <git@danielschaefer.me>
Date: Tue, 9 Apr 2019 04:02:49 +0200
Subject: [PATCH 05/18] nixos/cassandra: Fix test by listening on IP

Seems like you can't have a node as its own seed when it's listening on
an interface instead of an IP. At least the way it was done in the
test doesn't work and I can't figure out any other way than to just
listen on the IP address instead.
---
 nixos/tests/cassandra.nix | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/nixos/tests/cassandra.nix b/nixos/tests/cassandra.nix
index 60d0c6d76068..40d6b238dc02 100644
--- a/nixos/tests/cassandra.nix
+++ b/nixos/tests/cassandra.nix
@@ -2,25 +2,23 @@ import ./make-test.nix ({ pkgs, ...}:
 let
   # Change this to test a different version of Cassandra:
   testPackage = pkgs.cassandra;
-  cassandraCfg = 
+  cassandraCfg = hostname:
     { enable = true;
-      listenAddress = null;
-      listenInterface = "eth1";
-      rpcAddress = null;
-      rpcInterface = "eth1";
+      listenAddress = hostname;
+      rpcAddress = hostname;
       extraConfig =
         { start_native_transport = true;
           seed_provider =
             [{ class_name = "org.apache.cassandra.locator.SimpleSeedProvider";
-               parameters = [ { seeds = "cass0"; } ];
+               parameters = [ { seeds = "192.168.1.1"; } ];
             }];
         };
       package = testPackage;
     };
-  nodeCfg = extra: {pkgs, config, ...}:
+  nodeCfg = hostname: extra: {pkgs, config, ...}:
     { environment.systemPackages = [ testPackage ];
       networking.firewall.enable = false;
-      services.cassandra = cassandraCfg // extra;
+      services.cassandra = cassandraCfg hostname // extra;
       virtualisation.memorySize = 1024;
     };
 in
@@ -28,9 +26,9 @@ in
   name = "cassandra-ci";
 
   nodes = {
-    cass0 = nodeCfg {};
-    cass1 = nodeCfg {};
-    cass2 = nodeCfg { jvmOpts = [ "-Dcassandra.replace_address=cass1" ]; };
+    cass0 = nodeCfg "192.168.1.1" {};
+    cass1 = nodeCfg "192.168.1.2" {};
+    cass2 = nodeCfg "192.168.1.3" { jvmOpts = [ "-Dcassandra.replace_address=cass1" ]; };
   };
 
   testScript = ''

From d8ad5ad2a241049e49a527c1b24bf7f738ffb30c Mon Sep 17 00:00:00 2001
From: Daniel Schaefer <git@danielschaefer.me>
Date: Tue, 9 Apr 2019 06:31:55 +0200
Subject: [PATCH 06/18] cassandra: Properly wrap all binaries

Would previously overwrite the binary with the wrapper and thus wrap
itself (resulting in an infinite recursion on execution) for the
binaries in /bin.t
---
 pkgs/servers/nosql/cassandra/generic.nix | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/pkgs/servers/nosql/cassandra/generic.nix b/pkgs/servers/nosql/cassandra/generic.nix
index eaa85e69bec0..05f572c8caaa 100644
--- a/pkgs/servers/nosql/cassandra/generic.nix
+++ b/pkgs/servers/nosql/cassandra/generic.nix
@@ -23,7 +23,7 @@ stdenv.mkDerivation rec {
     url = "mirror://apache/cassandra/${version}/apache-${name}-bin.tar.gz";
   };
 
-  nativeBuildInputs = [ makeWrapper ];
+  nativeBuildInputs = [ makeWrapper coreutils ];
 
   installPhase = ''
     mkdir $out
@@ -51,8 +51,17 @@ stdenv.mkDerivation rec {
                bin/sstablescrub \
                bin/sstableupgrade \
                bin/sstableutil \
-               bin/sstableverify \
-               tools/bin/cassandra-stress \
+               bin/sstableverify; do
+      # Check if file exists because some don't exist across all versions
+      if [ -f $out/$cmd ]; then
+        wrapProgram $out/bin/$(basename "$cmd") \
+          --suffix-each LD_LIBRARY_PATH : ${libPath} \
+          --prefix PATH : ${binPath} \
+          --set JAVA_HOME ${jre}
+      fi
+    done
+
+    for cmd in tools/bin/cassandra-stress \
                tools/bin/cassandra-stressd \
                tools/bin/sstabledump \
                tools/bin/sstableexpiredblockers \
@@ -62,11 +71,9 @@ stdenv.mkDerivation rec {
                tools/bin/sstablerepairedset \
                tools/bin/sstablesplit \
                tools/bin/token-generator; do
-
-      # check if file exists because some bin tools don't exist across all
-      # cassandra versions
+      # Check if file exists because some don't exist across all versions
       if [ -f $out/$cmd ]; then
-        makeWrapper $out/$cmd $out/bin/$(${coreutils}/bin/basename "$cmd") \
+        makeWrapper $out/$cmd $out/bin/$(basename "$cmd") \
           --suffix-each LD_LIBRARY_PATH : ${libPath} \
           --prefix PATH : ${binPath} \
           --set JAVA_HOME ${jre}

From 2bcca9271acfbfb556ad47c4a11ad7bb5cd0f486 Mon Sep 17 00:00:00 2001
From: Daniel Schaefer <git@danielschaefer.me>
Date: Mon, 22 Apr 2019 18:06:40 +0200
Subject: [PATCH 07/18] nixos/cassandra: Reenable tests

---
 nixos/tests/all-tests.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix
index 8b38e802e62e..912e4f552313 100644
--- a/nixos/tests/all-tests.nix
+++ b/nixos/tests/all-tests.nix
@@ -36,6 +36,7 @@ in
   borgbackup = handleTest ./borgbackup.nix {};
   buildbot = handleTest ./buildbot.nix {};
   cadvisor = handleTestOn ["x86_64-linux"] ./cadvisor.nix {};
+  cassandra = handleTest ./cassandra.nix {};
   ceph = handleTestOn ["x86_64-linux"] ./ceph.nix {};
   certmgr = handleTest ./certmgr.nix {};
   cfssl = handleTestOn ["x86_64-linux"] ./cfssl.nix {};

From 35531f40164fe5876a484a7f213fb029fd2f927d Mon Sep 17 00:00:00 2001
From: Daniel Schaefer <git@danielschaefer.me>
Date: Tue, 23 Apr 2019 04:52:44 +0200
Subject: [PATCH 08/18] nixos/cassandra: Allow setting JMX credentials

If we have the ability to enable remote JMX we should also support
setting credentials for that because they become required if you turn it
on.
---
 .../modules/services/databases/cassandra.nix  | 38 ++++++++++++++++++-
 1 file changed, 37 insertions(+), 1 deletion(-)

diff --git a/nixos/modules/services/databases/cassandra.nix b/nixos/modules/services/databases/cassandra.nix
index 3c5a47682686..c15ac37be120 100644
--- a/nixos/modules/services/databases/cassandra.nix
+++ b/nixos/modules/services/databases/cassandra.nix
@@ -44,10 +44,25 @@ let
         ln -s "$cassandraLogbackConfig" "$out/logback.xml"
 
         cp "$cassandraEnvPkg" "$out/cassandra-env.sh"
+
         # Delete default JMX Port, otherwise we can't set it using env variable
         sed -i '/JMX_PORT="7199"/d' "$out/cassandra-env.sh"
+
+        # Delete default password file
+        sed -i '/-Dcom.sun.management.jmxremote.password.file=\/etc\/cassandra\/jmxremote.password/d' "$out/cassandra-env.sh"
       '';
     };
+  jmxPasswordFile = builtins.foldl'
+     (left: right: left + right) ""
+     (map (role: "${role.username} ${role.password}") cfg.jmxRoles);
+  fullJvmOptions = cfg.jvmOpts
+    ++ lib.optionals (cfg.jmxRoles != []) [
+      "-Dcom.sun.management.jmxremote.authenticate=true"
+      "-Dcom.sun.management.jmxremote.password.file=${pkgs.writeText "jmxremote.password" jmxPasswordFile}"
+    ]
+    ++ lib.optionals cfg.remoteJmx [
+      "-Djava.rmi.server.hostname=${cfg.rpcAddress}"
+    ];
 in {
   options.services.cassandra = {
     enable = mkEnableOption ''
@@ -322,6 +337,24 @@ in {
         Firewall it if needed.
       '';
     };
+    jmxRoles = mkOption {
+      default = [];
+      description = ''
+        Roles that are allowed to access the JMX (e.g. nodetool)
+      '';
+      type = types.listOf (types.submodule {
+        options = {
+          username = mkOption {
+            type = types.string;
+            description = "Username for JMX";
+          };
+          password = mkOption {
+            type = types.string;
+            description = "Password for JMX";
+          };
+        };
+      });
+    };
   };
 
   config = mkIf cfg.enable {
@@ -335,6 +368,9 @@ in {
         { assertion = (cfg.maxHeapSize == null) == (cfg.heapNewSize == null);
           message = "If you set either of maxHeapSize or heapNewSize you have to set both";
         }
+        { assertion = cfg.remoteJmx -> (cfg.jmxRoles != {});
+          message = "If you want JMX available remotely you need to set a password.";
+        }
       ];
     users = mkIf (cfg.user == defaultUser) {
       extraUsers."${defaultUser}" =
@@ -352,7 +388,7 @@ in {
         after = [ "network.target" ];
         environment =
           { CASSANDRA_CONF = "${cassandraEtc}";
-            JVM_OPTS = builtins.concatStringsSep " " cfg.jvmOpts;
+            JVM_OPTS = builtins.concatStringsSep " " fullJvmOptions;
             MAX_HEAP_SIZE = toString cfg.maxHeapSize;
             HEAP_NEWSIZE = toString cfg.heapNewSize;
             MALLOC_ARENA_MAX = toString cfg.mallocArenaMax;

From 4c880fd7426cafbc5091561d20062511e5eb967f Mon Sep 17 00:00:00 2001
From: Daniel Schaefer <git@danielschaefer.me>
Date: Tue, 23 Apr 2019 04:59:53 +0200
Subject: [PATCH 09/18] nixos/cassandra: Use ipAddress and capital letters

---
 nixos/tests/cassandra.nix | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/nixos/tests/cassandra.nix b/nixos/tests/cassandra.nix
index 40d6b238dc02..208e94c64c96 100644
--- a/nixos/tests/cassandra.nix
+++ b/nixos/tests/cassandra.nix
@@ -2,10 +2,11 @@ import ./make-test.nix ({ pkgs, ...}:
 let
   # Change this to test a different version of Cassandra:
   testPackage = pkgs.cassandra;
-  cassandraCfg = hostname:
+
+  cassandraCfg = ipAddress:
     { enable = true;
-      listenAddress = hostname;
-      rpcAddress = hostname;
+      listenAddress = ipAddress;
+      rpcAddress = ipAddress;
       extraConfig =
         { start_native_transport = true;
           seed_provider =
@@ -15,10 +16,10 @@ let
         };
       package = testPackage;
     };
-  nodeCfg = hostname: extra: {pkgs, config, ...}:
+  nodeCfg = ipAddress: extra: {pkgs, config, ...}:
     { environment.systemPackages = [ testPackage ];
       networking.firewall.enable = false;
-      services.cassandra = cassandraCfg hostname // extra;
+      services.cassandra = cassandraCfg ipAddress // extra;
       virtualisation.memorySize = 1024;
     };
 in
@@ -32,26 +33,29 @@ in
   };
 
   testScript = ''
-    subtest "timers exist", sub {
+    # Check configuration
+    subtest "Timers exist", sub {
       $cass0->succeed("systemctl list-timers | grep cassandra-full-repair.timer");
       $cass0->succeed("systemctl list-timers | grep cassandra-incremental-repair.timer");
     };
-    subtest "can connect via cqlsh", sub {
+    subtest "Can connect via cqlsh", sub {
       $cass0->waitForUnit("cassandra.service");
       $cass0->waitUntilSucceeds("nc -z cass0 9042");
       $cass0->succeed("echo 'show version;' | cqlsh cass0");
     };
-    subtest "nodetool is operational", sub {
+    subtest "Nodetool is operational", sub {
       $cass0->waitForUnit("cassandra.service");
       $cass0->waitUntilSucceeds("nc -z localhost 7199");
       $cass0->succeed("nodetool status --resolve-ip | egrep '^UN[[:space:]]+cass0'");
     };
-    subtest "bring up cluster", sub {
+
+    # Check cluster interaction
+    subtest "Bring up cluster", sub {
       $cass1->waitForUnit("cassandra.service");
       $cass1->waitUntilSucceeds("nodetool status | egrep -c '^UN' | grep 2");
       $cass0->succeed("nodetool status --resolve-ip | egrep '^UN[[:space:]]+cass1'");
     };
-    subtest "break and fix node", sub {
+    subtest "Break and fix node", sub {
       $cass1->block;
       $cass0->waitUntilSucceeds("nodetool status --resolve-ip | egrep -c '^DN[[:space:]]+cass1'");
       $cass0->succeed("nodetool status | egrep -c '^UN'  | grep 1");
@@ -59,7 +63,7 @@ in
       $cass1->waitUntilSucceeds("nodetool status | egrep -c '^UN'  | grep 2");
       $cass0->succeed("nodetool status | egrep -c '^UN'  | grep 2");
     };
-    subtest "replace crashed node", sub {
+    subtest "Replace crashed node", sub {
       $cass1->crash;
       $cass2->waitForUnit("cassandra.service");
       $cass0->waitUntilFails("nodetool status --resolve-ip | egrep '^UN[[:space:]]+cass1'");

From 2d014f444852f3ef240a6babf46cd501f8557c95 Mon Sep 17 00:00:00 2001
From: Daniel Schaefer <git@danielschaefer.me>
Date: Tue, 23 Apr 2019 05:02:30 +0200
Subject: [PATCH 10/18] nixos/cassandra: Test clusterName

---
 nixos/tests/cassandra.nix | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/nixos/tests/cassandra.nix b/nixos/tests/cassandra.nix
index 208e94c64c96..a5925e234c4d 100644
--- a/nixos/tests/cassandra.nix
+++ b/nixos/tests/cassandra.nix
@@ -2,9 +2,11 @@ import ./make-test.nix ({ pkgs, ...}:
 let
   # Change this to test a different version of Cassandra:
   testPackage = pkgs.cassandra;
+  clusterName = "NixOS Automated-Test Cluster";
 
   cassandraCfg = ipAddress:
     { enable = true;
+      inherit clusterName;
       listenAddress = ipAddress;
       rpcAddress = ipAddress;
       extraConfig =
@@ -48,6 +50,11 @@ in
       $cass0->waitUntilSucceeds("nc -z localhost 7199");
       $cass0->succeed("nodetool status --resolve-ip | egrep '^UN[[:space:]]+cass0'");
     };
+    subtest "Cluster name was set", sub {
+      $cass0->waitForUnit("cassandra.service");
+      $cass0->waitUntilSucceeds("nc -z localhost 7199");
+      $cass0->waitUntilSucceeds("nodetool describecluster | grep 'Name: ${clusterName}'");
+    };
 
     # Check cluster interaction
     subtest "Bring up cluster", sub {

From 545ac1820fcf556949f60f968e843759e5b1225b Mon Sep 17 00:00:00 2001
From: Daniel Schaefer <git@danielschaefer.me>
Date: Tue, 23 Apr 2019 05:03:07 +0200
Subject: [PATCH 11/18] nixos/cassandra: Test seedAddresses

---
 nixos/tests/cassandra.nix | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/nixos/tests/cassandra.nix b/nixos/tests/cassandra.nix
index a5925e234c4d..6779a0172dbd 100644
--- a/nixos/tests/cassandra.nix
+++ b/nixos/tests/cassandra.nix
@@ -9,13 +9,7 @@ let
       inherit clusterName;
       listenAddress = ipAddress;
       rpcAddress = ipAddress;
-      extraConfig =
-        { start_native_transport = true;
-          seed_provider =
-            [{ class_name = "org.apache.cassandra.locator.SimpleSeedProvider";
-               parameters = [ { seeds = "192.168.1.1"; } ];
-            }];
-        };
+      seedAddresses = [ "192.168.1.1" ];
       package = testPackage;
     };
   nodeCfg = ipAddress: extra: {pkgs, config, ...}:

From 2368345052adcf0076628ff7a732cc99c017a2c3 Mon Sep 17 00:00:00 2001
From: Daniel Schaefer <git@danielschaefer.me>
Date: Tue, 23 Apr 2019 05:03:41 +0200
Subject: [PATCH 12/18] nixos/cassandra: Set test VM IPs statically

---
 nixos/tests/cassandra.nix | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/nixos/tests/cassandra.nix b/nixos/tests/cassandra.nix
index 6779a0172dbd..eab757b80a54 100644
--- a/nixos/tests/cassandra.nix
+++ b/nixos/tests/cassandra.nix
@@ -14,7 +14,13 @@ let
     };
   nodeCfg = ipAddress: extra: {pkgs, config, ...}:
     { environment.systemPackages = [ testPackage ];
-      networking.firewall.enable = false;
+      networking = {
+        firewall.allowedTCPPorts = [ 7000 7199 9042 ];
+        useDHCP = false;
+        interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [
+          { address = ipAddress; prefixLength = 24; }
+        ];
+      };
       services.cassandra = cassandraCfg ipAddress // extra;
       virtualisation.memorySize = 1024;
     };

From 3162f45388638d04faac82a5d8c2dcf3076a84bc Mon Sep 17 00:00:00 2001
From: Daniel Schaefer <git@danielschaefer.me>
Date: Tue, 23 Apr 2019 05:05:06 +0200
Subject: [PATCH 13/18] nixos/cassandra: Test maxHeapSize

---
 nixos/tests/cassandra.nix | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/nixos/tests/cassandra.nix b/nixos/tests/cassandra.nix
index eab757b80a54..a14cec5ab19a 100644
--- a/nixos/tests/cassandra.nix
+++ b/nixos/tests/cassandra.nix
@@ -4,6 +4,14 @@ let
   testPackage = pkgs.cassandra;
   clusterName = "NixOS Automated-Test Cluster";
 
+  numMaxHeapSize = "400";
+  getHeapLimitCommand = ''
+    nodetool info | grep "^Heap Memory" | awk \'{print $NF}\'
+  '';
+  checkHeapLimitCommand = ''
+    [ 1 -eq "$(echo "$(${getHeapLimitCommand}) < ${numMaxHeapSize}" | ${pkgs.bc}/bin/bc)" ]
+  '';
+
   cassandraCfg = ipAddress:
     { enable = true;
       inherit clusterName;
@@ -11,6 +19,8 @@ let
       rpcAddress = ipAddress;
       seedAddresses = [ "192.168.1.1" ];
       package = testPackage;
+      maxHeapSize = "${numMaxHeapSize}M";
+      heapNewSize = "100M";
     };
   nodeCfg = ipAddress: extra: {pkgs, config, ...}:
     { environment.systemPackages = [ testPackage ];
@@ -55,6 +65,11 @@ in
       $cass0->waitUntilSucceeds("nc -z localhost 7199");
       $cass0->waitUntilSucceeds("nodetool describecluster | grep 'Name: ${clusterName}'");
     };
+    subtest "Heap limit set correctly", sub {
+      # Nodetool takes a while until it can display info
+      $cass0->waitUntilSucceeds('nodetool info');
+      $cass0->succeed('${checkHeapLimitCommand}');
+    };
 
     # Check cluster interaction
     subtest "Bring up cluster", sub {

From 7d646f260566361606b256b608fdf57e9d8a4564 Mon Sep 17 00:00:00 2001
From: Daniel Schaefer <git@danielschaefer.me>
Date: Tue, 23 Apr 2019 05:05:39 +0200
Subject: [PATCH 14/18] nixos/cassandra: Test jmxRoles

---
 nixos/tests/cassandra.nix | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/nixos/tests/cassandra.nix b/nixos/tests/cassandra.nix
index a14cec5ab19a..c67239931e10 100644
--- a/nixos/tests/cassandra.nix
+++ b/nixos/tests/cassandra.nix
@@ -4,6 +4,11 @@ let
   testPackage = pkgs.cassandra;
   clusterName = "NixOS Automated-Test Cluster";
 
+  jmxRoles = [{ username = "me"; password = "password"; }];
+  jmxAuthArgs = "-u ${(builtins.elemAt jmxRoles 0).username} -pw ${(builtins.elemAt jmxRoles 0).password}";
+  clusterName = "NixOS Test Cluster";
+
+  # Would usually be assigned to 512M
   numMaxHeapSize = "400";
   getHeapLimitCommand = ''
     nodetool info | grep "^Heap Memory" | awk \'{print $NF}\'
@@ -40,7 +45,7 @@ in
 
   nodes = {
     cass0 = nodeCfg "192.168.1.1" {};
-    cass1 = nodeCfg "192.168.1.2" {};
+    cass1 = nodeCfg "192.168.1.2" { remoteJmx = true; inherit jmxRoles; };
     cass2 = nodeCfg "192.168.1.3" { jvmOpts = [ "-Dcassandra.replace_address=cass1" ]; };
   };
 
@@ -74,15 +79,25 @@ in
     # Check cluster interaction
     subtest "Bring up cluster", sub {
       $cass1->waitForUnit("cassandra.service");
-      $cass1->waitUntilSucceeds("nodetool status | egrep -c '^UN' | grep 2");
+      $cass1->waitUntilSucceeds("nodetool ${jmxAuthArgs} status | egrep -c '^UN' | grep 2");
       $cass0->succeed("nodetool status --resolve-ip | egrep '^UN[[:space:]]+cass1'");
     };
+    subtest "Remote authenticated jmx", sub {
+      # Doesn't work if not enabled
+      $cass0->waitUntilSucceeds("nc -z localhost 7199");
+      $cass1->fail("nc -z 192.168.1.1 7199");
+      $cass1->fail("nodetool -h 192.168.1.1 status");
+
+      # Works if enabled
+      $cass1->waitUntilSucceeds("nc -z localhost 7199");
+      $cass0->succeed("nodetool -h 192.168.1.2 ${jmxAuthArgs} status");
+    };
     subtest "Break and fix node", sub {
       $cass1->block;
       $cass0->waitUntilSucceeds("nodetool status --resolve-ip | egrep -c '^DN[[:space:]]+cass1'");
       $cass0->succeed("nodetool status | egrep -c '^UN'  | grep 1");
       $cass1->unblock;
-      $cass1->waitUntilSucceeds("nodetool status | egrep -c '^UN'  | grep 2");
+      $cass1->waitUntilSucceeds("nodetool ${jmxAuthArgs} status | egrep -c '^UN'  | grep 2");
       $cass0->succeed("nodetool status | egrep -c '^UN'  | grep 2");
     };
     subtest "Replace crashed node", sub {

From 9ecd58478595dab0270c20309e0b0399711b3a0f Mon Sep 17 00:00:00 2001
From: Daniel Schaefer <git@danielschaefer.me>
Date: Fri, 26 Apr 2019 00:59:23 +0200
Subject: [PATCH 15/18] nixos/cassandra: Add option for password file path

If you're on a multi user system you don't want to have the password in
the nix-store. With the new jmxRolesFile option you can specify your own
protected file instead.
---
 .../modules/services/databases/cassandra.nix  | 30 +++++++++++++++----
 1 file changed, 25 insertions(+), 5 deletions(-)

diff --git a/nixos/modules/services/databases/cassandra.nix b/nixos/modules/services/databases/cassandra.nix
index c15ac37be120..d6071b6ee675 100644
--- a/nixos/modules/services/databases/cassandra.nix
+++ b/nixos/modules/services/databases/cassandra.nix
@@ -19,7 +19,7 @@ let
        data_file_directories = [ "${cfg.homeDir}/data" ];
        commitlog_directory = "${cfg.homeDir}/commitlog";
        saved_caches_directory = "${cfg.homeDir}/saved_caches";
-     } // (if builtins.compareVersions cfg.package.version "3" >= 0
+     } // (if lib.versionAtLeast cfg.package.version "3"
              then { hints_directory = "${cfg.homeDir}/hints"; }
              else {})
     );
@@ -52,13 +52,13 @@ let
         sed -i '/-Dcom.sun.management.jmxremote.password.file=\/etc\/cassandra\/jmxremote.password/d' "$out/cassandra-env.sh"
       '';
     };
-  jmxPasswordFile = builtins.foldl'
+  defaultJmxRolesFile = builtins.foldl'
      (left: right: left + right) ""
      (map (role: "${role.username} ${role.password}") cfg.jmxRoles);
   fullJvmOptions = cfg.jvmOpts
     ++ lib.optionals (cfg.jmxRoles != []) [
       "-Dcom.sun.management.jmxremote.authenticate=true"
-      "-Dcom.sun.management.jmxremote.password.file=${pkgs.writeText "jmxremote.password" jmxPasswordFile}"
+      "-Dcom.sun.management.jmxremote.password.file=${cfg.jmxRolesFile}"
     ]
     ++ lib.optionals cfg.remoteJmx [
       "-Djava.rmi.server.hostname=${cfg.rpcAddress}"
@@ -341,6 +341,10 @@ in {
       default = [];
       description = ''
         Roles that are allowed to access the JMX (e.g. nodetool)
+        BEWARE: The passwords will be stored world readable in the nix-store.
+
+        Doesn't work in versions older than 3.11 because they don't like that
+        it's world readable.
       '';
       type = types.listOf (types.submodule {
         options = {
@@ -355,6 +359,19 @@ in {
         };
       });
     };
+    jmxRolesFile = mkOption {
+      type = types.nullOr types.path;
+      default = if (lib.versionAtLeast cfg.package.version "3.11")
+                then pkgs.writeText "jmx-roles-file" defaultJmxRolesFile
+                else null;
+      example = "/var/lib/cassandra/jmx.password";
+      description = ''
+        Specify your own jmx roles file.
+
+        Make sure the permissions forbid "others" from reading the file if
+        you're using Cassandra below version 3.11.
+      '';
+    };
   };
 
   config = mkIf cfg.enable {
@@ -368,8 +385,11 @@ in {
         { assertion = (cfg.maxHeapSize == null) == (cfg.heapNewSize == null);
           message = "If you set either of maxHeapSize or heapNewSize you have to set both";
         }
-        { assertion = cfg.remoteJmx -> (cfg.jmxRoles != {});
-          message = "If you want JMX available remotely you need to set a password.";
+        { assertion = cfg.remoteJmx -> cfg.jmxRolesFile != null;
+          message = ''
+            If you want JMX available remotely you need to set a password using
+            `jmxRoles` or `jmxRolesFile` if using Cassandra older than v3.11.
+          '';
         }
       ];
     users = mkIf (cfg.user == defaultUser) {

From 79f7f89442d9132c35b6417187cb1567eca2b910 Mon Sep 17 00:00:00 2001
From: Daniel Schaefer <git@danielschaefer.me>
Date: Fri, 26 Apr 2019 01:13:09 +0200
Subject: [PATCH 16/18] nixos/cassandra: Use docbook instead of markdown style

---
 nixos/modules/services/databases/cassandra.nix | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/nixos/modules/services/databases/cassandra.nix b/nixos/modules/services/databases/cassandra.nix
index d6071b6ee675..452691e42322 100644
--- a/nixos/modules/services/databases/cassandra.nix
+++ b/nixos/modules/services/databases/cassandra.nix
@@ -229,11 +229,11 @@ in {
       example = literalExample "null";
       description = ''
           Set the interval how often full repairs are run, i.e.
-          `nodetool repair --full` is executed. See
+          <literal>nodetool repair --full</literal> is executed. See
           https://cassandra.apache.org/doc/latest/operating/repair.html
           for more information.
 
-          Set to `null` to disable full repairs.
+          Set to <literal>null</literal> to disable full repairs.
         '';
     };
     fullRepairOptions = mkOption {
@@ -250,11 +250,11 @@ in {
       example = literalExample "null";
       description = ''
           Set the interval how often incremental repairs are run, i.e.
-          `nodetool repair` is executed. See
+          <literal>nodetool repair<literal> is executed. See
           https://cassandra.apache.org/doc/latest/operating/repair.html
           for more information.
 
-          Set to `null` to disable incremental repairs.
+          Set to <literal>null</literal> to disable incremental repairs.
         '';
     };
     incrementalRepairOptions = mkOption {
@@ -342,6 +342,8 @@ in {
       description = ''
         Roles that are allowed to access the JMX (e.g. nodetool)
         BEWARE: The passwords will be stored world readable in the nix-store.
+                It's recommended to use your own protected file using
+                <literal>jmxRolesFile</literal>
 
         Doesn't work in versions older than 3.11 because they don't like that
         it's world readable.
@@ -388,7 +390,8 @@ in {
         { assertion = cfg.remoteJmx -> cfg.jmxRolesFile != null;
           message = ''
             If you want JMX available remotely you need to set a password using
-            `jmxRoles` or `jmxRolesFile` if using Cassandra older than v3.11.
+            <literal>jmxRoles</literal> or <literal>jmxRolesFile</literal> if
+            using Cassandra older than v3.11.
           '';
         }
       ];

From 8e5ba87b36c8640ada623811ec5f22af6fe60d44 Mon Sep 17 00:00:00 2001
From: Daniel Schaefer <git@danielschaefer.me>
Date: Fri, 26 Apr 2019 01:13:26 +0200
Subject: [PATCH 17/18] nixos/cassandra: Test remote auth only on v3.11+

Below that it works but only when supplying a custom password file with
restricted permissions (i.e. outside the nix-store). We can't do that
using an absolute path in the tests.
---
 nixos/tests/cassandra.nix | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/nixos/tests/cassandra.nix b/nixos/tests/cassandra.nix
index c67239931e10..aea4fa4d1c95 100644
--- a/nixos/tests/cassandra.nix
+++ b/nixos/tests/cassandra.nix
@@ -1,12 +1,13 @@
-import ./make-test.nix ({ pkgs, ...}:
+import ./make-test.nix ({ pkgs, lib, ... }:
 let
   # Change this to test a different version of Cassandra:
   testPackage = pkgs.cassandra;
   clusterName = "NixOS Automated-Test Cluster";
 
+  testRemoteAuth = lib.versionAtLeast testPackage.version "3.11";
   jmxRoles = [{ username = "me"; password = "password"; }];
+  jmxRolesFile = ./cassandra-jmx-roles;
   jmxAuthArgs = "-u ${(builtins.elemAt jmxRoles 0).username} -pw ${(builtins.elemAt jmxRoles 0).password}";
-  clusterName = "NixOS Test Cluster";
 
   # Would usually be assigned to 512M
   numMaxHeapSize = "400";
@@ -45,7 +46,7 @@ in
 
   nodes = {
     cass0 = nodeCfg "192.168.1.1" {};
-    cass1 = nodeCfg "192.168.1.2" { remoteJmx = true; inherit jmxRoles; };
+    cass1 = nodeCfg "192.168.1.2" (lib.optionalAttrs testRemoteAuth { inherit jmxRoles; remoteJmx = true; });
     cass2 = nodeCfg "192.168.1.3" { jvmOpts = [ "-Dcassandra.replace_address=cass1" ]; };
   };
 
@@ -82,6 +83,7 @@ in
       $cass1->waitUntilSucceeds("nodetool ${jmxAuthArgs} status | egrep -c '^UN' | grep 2");
       $cass0->succeed("nodetool status --resolve-ip | egrep '^UN[[:space:]]+cass1'");
     };
+  '' + lib.optionalString testRemoteAuth ''
     subtest "Remote authenticated jmx", sub {
       # Doesn't work if not enabled
       $cass0->waitUntilSucceeds("nc -z localhost 7199");
@@ -92,6 +94,7 @@ in
       $cass1->waitUntilSucceeds("nc -z localhost 7199");
       $cass0->succeed("nodetool -h 192.168.1.2 ${jmxAuthArgs} status");
     };
+  '' + ''
     subtest "Break and fix node", sub {
       $cass1->block;
       $cass0->waitUntilSucceeds("nodetool status --resolve-ip | egrep -c '^DN[[:space:]]+cass1'");

From 03503121da3305b03b17d9f4fbfe99258cfd195f Mon Sep 17 00:00:00 2001
From: Daniel Schaefer <git@danielschaefer.me>
Date: Thu, 2 May 2019 08:59:39 +0200
Subject: [PATCH 18/18] nixos/cassandra: Don't force SimpleSeedProvider

If the `seedAddresses` is not set, don't force `SimpleSeedProvider` to
be in `seed_provider`. This could cause problems in a multi-datacenter
deployment when a different seed provider is preferred.
---
 nixos/modules/services/databases/cassandra.nix | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/nixos/modules/services/databases/cassandra.nix b/nixos/modules/services/databases/cassandra.nix
index 452691e42322..fc5812c49df0 100644
--- a/nixos/modules/services/databases/cassandra.nix
+++ b/nixos/modules/services/databases/cassandra.nix
@@ -12,16 +12,17 @@ let
        cluster_name = cfg.clusterName;
        partitioner = "org.apache.cassandra.dht.Murmur3Partitioner";
        endpoint_snitch = "SimpleSnitch";
-       seed_provider =
-         [{ class_name = "org.apache.cassandra.locator.SimpleSeedProvider";
-            parameters = [ { seeds = concatStringsSep "," cfg.seedAddresses; } ];
-         }];
        data_file_directories = [ "${cfg.homeDir}/data" ];
        commitlog_directory = "${cfg.homeDir}/commitlog";
        saved_caches_directory = "${cfg.homeDir}/saved_caches";
-     } // (if lib.versionAtLeast cfg.package.version "3"
-             then { hints_directory = "${cfg.homeDir}/hints"; }
-             else {})
+     } // (lib.optionalAttrs (cfg.seedAddresses != []) {
+       seed_provider = [{
+         class_name = "org.apache.cassandra.locator.SimpleSeedProvider";
+         parameters = [ { seeds = concatStringsSep "," cfg.seedAddresses; } ];
+       }];
+     }) // (lib.optionalAttrs (lib.versionAtLeast cfg.package.version "3") {
+       hints_directory = "${cfg.homeDir}/hints";
+     })
     );
   cassandraConfigWithAddresses = cassandraConfig //
     ( if cfg.listenAddress == null