nixpkgs/nixos/tests/etcd/etcd-cluster.nix

# This test runs simple etcd cluster

import ../make-test-python.nix (
  { pkgs, ... }:
  let

    runWithOpenSSL =
      file: cmd:
      pkgs.runCommand file {
        buildInputs = [ pkgs.openssl ];
      } cmd;

    ca_key = runWithOpenSSL "ca-key.pem" "openssl genrsa -out $out 2048";
    ca_pem = runWithOpenSSL "ca.pem" ''
      openssl req \
        -x509 -new -nodes -key ${ca_key} \
        -days 10000 -out $out -subj "/CN=etcd-ca"
    '';
    etcd_key = runWithOpenSSL "etcd-key.pem" "openssl genrsa -out $out 2048";
    etcd_csr = runWithOpenSSL "etcd.csr" ''
      openssl req \
         -new -key ${etcd_key} \
         -out $out -subj "/CN=etcd" \
         -config ${openssl_cnf}
    '';
    etcd_cert = runWithOpenSSL "etcd.pem" ''
      openssl x509 \
        -req -in ${etcd_csr} \
        -CA ${ca_pem} -CAkey ${ca_key} \
        -CAcreateserial -out $out \
        -days 365 -extensions v3_req \
        -extfile ${openssl_cnf}
    '';

    etcd_client_key = runWithOpenSSL "etcd-client-key.pem" "openssl genrsa -out $out 2048";

    etcd_client_csr = runWithOpenSSL "etcd-client-key.pem" ''
      openssl req \
        -new -key ${etcd_client_key} \
        -out $out -subj "/CN=etcd-client" \
        -config ${client_openssl_cnf}
    '';

    etcd_client_cert = runWithOpenSSL "etcd-client.crt" ''
      openssl x509 \
        -req -in ${etcd_client_csr} \
        -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
        -out $out -days 365 -extensions v3_req \
        -extfile ${client_openssl_cnf}
    '';

    openssl_cnf = pkgs.writeText "openssl.cnf" ''
      ions = v3_req
      distinguished_name = req_distinguished_name
      [req_distinguished_name]
      [ v3_req ]
      basicConstraints = CA:FALSE
      keyUsage = digitalSignature, keyEncipherment
      extendedKeyUsage = serverAuth, clientAuth
      subjectAltName = @alt_names
      [alt_names]
      DNS.1 = node1
      DNS.2 = node2
      DNS.3 = node3
      IP.1 = 127.0.0.1
    '';

    client_openssl_cnf = pkgs.writeText "client-openssl.cnf" ''
      ions = v3_req
      distinguished_name = req_distinguished_name
      [req_distinguished_name]
      [ v3_req ]
      basicConstraints = CA:FALSE
      keyUsage = digitalSignature, keyEncipherment
      extendedKeyUsage = clientAuth
    '';

    nodeConfig = {
      services = {
        etcd = {
          enable = true;
          keyFile = etcd_key;
          certFile = etcd_cert;
          trustedCaFile = ca_pem;
          clientCertAuth = true;
          listenClientUrls = [ "https://127.0.0.1:2379" ];
          listenPeerUrls = [ "https://0.0.0.0:2380" ];
        };
      };

      environment.variables = {
        ETCD_CERT_FILE = "${etcd_client_cert}";
        ETCD_KEY_FILE = "${etcd_client_key}";
        ETCD_CA_FILE = "${ca_pem}";
        ETCDCTL_ENDPOINTS = "https://127.0.0.1:2379";
        ETCDCTL_CACERT = "${ca_pem}";
        ETCDCTL_CERT = "${etcd_cert}";
        ETCDCTL_KEY = "${etcd_key}";
      };

      networking.firewall.allowedTCPPorts = [ 2380 ];
    };
  in
  {
    name = "etcd-cluster";

    meta = with pkgs.lib.maintainers; {
      maintainers = [ offline ];
    };

    nodes = {
      node1 =
        { ... }:
        {
          require = [ nodeConfig ];
          services.etcd = {
            initialCluster = [
              "node1=https://node1:2380"
              "node2=https://node2:2380"
            ];
            initialAdvertisePeerUrls = [ "https://node1:2380" ];
          };
        };

      node2 =
        { ... }:
        {
          require = [ nodeConfig ];
          services.etcd = {
            initialCluster = [
              "node1=https://node1:2380"
              "node2=https://node2:2380"
            ];
            initialAdvertisePeerUrls = [ "https://node2:2380" ];
          };
        };

      node3 =
        { ... }:
        {
          require = [ nodeConfig ];
          services.etcd = {
            initialCluster = [
              "node1=https://node1:2380"
              "node2=https://node2:2380"
              "node3=https://node3:2380"
            ];
            initialAdvertisePeerUrls = [ "https://node3:2380" ];
            initialClusterState = "existing";
          };
        };
    };

    testScript = ''
      with subtest("should start etcd cluster"):
          node1.start()
          node2.start()
          node1.wait_for_unit("etcd.service")
          node2.wait_for_unit("etcd.service")
          node2.wait_until_succeeds("etcdctl endpoint status")
          node1.succeed("etcdctl put /foo/bar 'Hello world'")
          node2.succeed("etcdctl get /foo/bar | grep 'Hello world'")

      with subtest("should add another member"):
          node1.wait_until_succeeds("etcdctl member add node3 --peer-urls=https://node3:2380")
          node3.start()
          node3.wait_for_unit("etcd.service")
          node3.wait_until_succeeds("etcdctl member list | grep 'node3'")
          node3.succeed("etcdctl endpoint status")

      with subtest("should survive member crash"):
          node3.crash()
          node1.succeed("etcdctl endpoint status")
          node1.succeed("etcdctl put /foo/bar 'Hello degraded world'")
          node1.succeed("etcdctl get /foo/bar | grep 'Hello degraded world'")
    '';
  }
)