nixpkgs/nixos/tests/kubernetes.nix

{ system ? builtins.currentSystem }:

with import ../lib/testing.nix { inherit system; };
with import ../lib/qemu-flags.nix;
with pkgs.lib;

let
  redisPod = pkgs.writeText "redis-master-pod.json" (builtins.toJSON {
    kind = "Pod";
    apiVersion = "v1";
    metadata.name = "redis";
    metadata.labels.name = "redis";
    spec.containers = [{
      name = "redis";
      image = "redis";
      args = ["--bind" "0.0.0.0"];
      imagePullPolicy = "Never";
      ports = [{
        name = "redis-server";
        containerPort = 6379;
      }];
    }];
  });

  redisService = pkgs.writeText "redis-service.json" (builtins.toJSON {
    kind = "Service";
    apiVersion = "v1";
    metadata.name = "redis";
    spec = {
      ports = [{port = 6379; targetPort = 6379;}];
      selector = {name = "redis";};
    };
  });

  redisImage = pkgs.dockerTools.buildImage {
    name = "redis";
    tag = "latest";
    contents = pkgs.redis;
    config.Entrypoint = "/bin/redis-server";
  };

  testSimplePod = ''
    $kubernetes->execute("docker load < ${redisImage}");
    $kubernetes->waitUntilSucceeds("kubectl create -f ${redisPod}");
    $kubernetes->succeed("kubectl create -f ${redisService}");
    $kubernetes->waitUntilSucceeds("kubectl get pod redis | grep Running");
    $kubernetes->succeed("nc -z \$\(dig \@10.10.0.1 redis.default.svc.cluster.local +short\) 6379");
  '';
in {
  # This test runs kubernetes on a single node
  trivial = makeTest {
    name = "kubernetes-trivial";

    nodes = {
      kubernetes =
        { config, pkgs, lib, nodes, ... }:
          {
            virtualisation.memorySize = 768;
            virtualisation.diskSize = 2048;

            programs.bash.enableCompletion = true;
            environment.systemPackages = with pkgs; [ netcat bind ];

            services.kubernetes.roles = ["master" "node"];
            virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false -b cbr0";

            networking.bridges.cbr0.interfaces = [];
            networking.interfaces.cbr0 = {};
          };
    };

    testScript = ''
      startAll;

      $kubernetes->waitUntilSucceeds("kubectl get nodes | grep kubernetes | grep Ready");

      ${testSimplePod}
    '';
  };

  cluster = let
    runWithOpenSSL = file: cmd: pkgs.runCommand file {
      buildInputs = [ pkgs.openssl ];
    } cmd;

    ca_key = runWithOpenSSL "ca-key.pem" "openssl genrsa -out $out 2048";
    ca_pem = runWithOpenSSL "ca.pem" ''
      openssl req \
        -x509 -new -nodes -key ${ca_key} \
        -days 10000 -out $out -subj "/CN=etcd-ca"
    '';
    etcd_key = runWithOpenSSL "etcd-key.pem" "openssl genrsa -out $out 2048";
    etcd_csr = runWithOpenSSL "etcd.csr" ''
      openssl req \
        -new -key ${etcd_key} \
        -out $out -subj "/CN=etcd" \
        -config ${openssl_cnf}
    '';
    etcd_cert = runWithOpenSSL "etcd.pem" ''
      openssl x509 \
        -req -in ${etcd_csr} \
        -CA ${ca_pem} -CAkey ${ca_key} \
        -CAcreateserial -out $out \
        -days 365 -extensions v3_req \
        -extfile ${openssl_cnf}
    '';

    etcd_client_key = runWithOpenSSL "etcd-client-key.pem"
      "openssl genrsa -out $out 2048";

    etcd_client_csr = runWithOpenSSL "etcd-client-key.pem" ''
      openssl req \
        -new -key ${etcd_client_key} \
        -out $out -subj "/CN=etcd-client" \
        -config ${client_openssl_cnf}
    '';

    etcd_client_cert = runWithOpenSSL "etcd-client.crt" ''
      openssl x509 \
        -req -in ${etcd_client_csr} \
        -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
        -out $out -days 365 -extensions v3_req \
        -extfile ${client_openssl_cnf}
    '';

    apiserver_key = runWithOpenSSL "apiserver-key.pem" "openssl genrsa -out $out 2048";

    apiserver_csr = runWithOpenSSL "apiserver.csr" ''
      openssl req \
        -new -key ${apiserver_key} \
        -out $out -subj "/CN=kube-apiserver" \
        -config ${apiserver_cnf}
    '';

    apiserver_cert = runWithOpenSSL "apiserver.pem" ''
      openssl x509 \
        -req -in ${apiserver_csr} \
        -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
        -out $out -days 365 -extensions v3_req \
        -extfile ${apiserver_cnf}
    '';

    worker_key = runWithOpenSSL "worker-key.pem" "openssl genrsa -out $out 2048";

    worker_csr = runWithOpenSSL "worker.csr" ''
      openssl req \
        -new -key ${worker_key} \
        -out $out -subj "/CN=kube-worker" \
        -config ${worker_cnf}
    '';

    worker_cert = runWithOpenSSL "worker.pem" ''
      openssl x509 \
        -req -in ${worker_csr} \
        -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
        -out $out -days 365 -extensions v3_req \
        -extfile ${worker_cnf}
    '';

    openssl_cnf = pkgs.writeText "openssl.cnf" ''
      [req]
      req_extensions = v3_req
      distinguished_name = req_distinguished_name
      [req_distinguished_name]
      [ v3_req ]
      basicConstraints = CA:FALSE
      keyUsage = digitalSignature, keyEncipherment
      extendedKeyUsage = serverAuth
      subjectAltName = @alt_names
      [alt_names]
      DNS.1 = etcd1
      DNS.2 = etcd2
      DNS.3 = etcd3
      IP.1 = 127.0.0.1
    '';

    client_openssl_cnf = pkgs.writeText "client-openssl.cnf" ''
      [req]
      req_extensions = v3_req
      distinguished_name = req_distinguished_name
      [req_distinguished_name]
      [ v3_req ]
      basicConstraints = CA:FALSE
      keyUsage = digitalSignature, keyEncipherment
      extendedKeyUsage = clientAuth
    '';

    apiserver_cnf = pkgs.writeText "apiserver-openssl.cnf" ''
      [req]
      req_extensions = v3_req
      distinguished_name = req_distinguished_name
      [req_distinguished_name]
      [ v3_req ]
      basicConstraints = CA:FALSE
      keyUsage = nonRepudiation, digitalSignature, keyEncipherment
      subjectAltName = @alt_names
      [alt_names]
      DNS.1 = kubernetes
      DNS.2 = kubernetes.default
      DNS.3 = kubernetes.default.svc
      DNS.4 = kubernetes.default.svc.cluster.local
      IP.1 = 10.10.10.1
    '';

    worker_cnf = pkgs.writeText "worker-openssl.cnf" ''
      [req]
      req_extensions = v3_req
      distinguished_name = req_distinguished_name
      [req_distinguished_name]
      [ v3_req ]
      basicConstraints = CA:FALSE
      keyUsage = nonRepudiation, digitalSignature, keyEncipherment
      subjectAltName = @alt_names
      [alt_names]
      DNS.1 = kubeWorker1
      DNS.2 = kubeWorker2
    '';

    etcdNodeConfig = {
      virtualisation.memorySize = 128;

      services = {
        etcd = {
          enable = true;
          keyFile = etcd_key;
          certFile = etcd_cert;
          trustedCaFile = ca_pem;
          peerClientCertAuth = true;
          listenClientUrls = ["https://0.0.0.0:2379"];
          listenPeerUrls = ["https://0.0.0.0:2380"];
        };
      };

      environment.variables = {
        ETCDCTL_CERT_FILE = "${etcd_client_cert}";
        ETCDCTL_KEY_FILE = "${etcd_client_key}";
        ETCDCTL_CA_FILE = "${ca_pem}";
        ETCDCTL_PEERS = "https://127.0.0.1:2379";
      };

      networking.firewall.allowedTCPPorts = [ 2379 2380 ];
    };

    kubeConfig = {
      virtualisation.diskSize = 2048;
      programs.bash.enableCompletion = true;

      services.flannel = {
        enable = true;
        network = "10.10.0.0/16";
        iface = "eth1";
        etcd = {
          endpoints = ["https://etcd1:2379" "https://etcd2:2379" "https://etcd3:2379"];
          keyFile = etcd_client_key;
          certFile = etcd_client_cert;
          caFile = ca_pem;
        };
      };

      # vxlan
      networking.firewall.allowedUDPPorts = [ 8472 ];

      systemd.services.docker.after = ["flannel.service"];
      systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/subnet.env";
      virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --bip $FLANNEL_SUBNET";

      services.kubernetes.verbose = true;
      services.kubernetes.etcd = {
        servers = ["https://etcd1:2379" "https://etcd2:2379" "https://etcd3:2379"];
        keyFile = etcd_client_key;
        certFile = etcd_client_cert;
        caFile = ca_pem;
      };

      environment.systemPackages = [ pkgs.bind pkgs.tcpdump pkgs.utillinux ];
    };

    kubeMasterConfig = {pkgs, ...}: {
      require = [kubeConfig];

      # kube apiserver
      networking.firewall.allowedTCPPorts = [ 443 ];

      virtualisation.memorySize = 512;

      services.kubernetes = {
        roles = ["master"];
        scheduler.leaderElect = true;
        controllerManager.leaderElect = true;

        apiserver = {
          publicAddress = "0.0.0.0";
          advertiseAddress = "192.168.1.8";
          tlsKeyFile = apiserver_key;
          tlsCertFile = apiserver_cert;
          clientCaFile = ca_pem;
          kubeletClientCaFile = ca_pem;
          kubeletClientKeyFile = worker_key;
          kubeletClientCertFile = worker_cert;
        };
      };
    };

    kubeWorkerConfig = { pkgs, ... }: {
      require = [kubeConfig];

      virtualisation.memorySize = 512;

      # kubelet
      networking.firewall.allowedTCPPorts = [ 10250 ];

      services.kubernetes = {
        roles = ["node"];
        kubeconfig = {
          server = "https://kubernetes:443";
          caFile = ca_pem;
          certFile = worker_cert;
          keyFile = worker_key;
        };
        kubelet = {
          tlsKeyFile = worker_key;
          tlsCertFile = worker_cert;
        };
      };
    };
  in makeTest {
    name = "kubernetes-cluster";

    nodes = {
      etcd1 = { config, pkgs, nodes, ... }: {
        require = [etcdNodeConfig];
        services.etcd = {
          advertiseClientUrls = ["https://etcd1:2379"];
          initialCluster = ["etcd1=https://etcd1:2380" "etcd2=https://etcd2:2380" "etcd3=https://etcd3:2380"];
          initialAdvertisePeerUrls = ["https://etcd1:2380"];
        };
      };

      etcd2 = { config, pkgs, ... }: {
        require = [etcdNodeConfig];
        services.etcd = {
          advertiseClientUrls = ["https://etcd2:2379"];
          initialCluster = ["etcd1=https://etcd1:2380" "etcd2=https://etcd2:2380" "etcd3=https://etcd3:2380"];
          initialAdvertisePeerUrls = ["https://etcd2:2380"];
        };
      };

      etcd3 = { config, pkgs, ... }: {
        require = [etcdNodeConfig];
        services.etcd = {
          advertiseClientUrls = ["https://etcd3:2379"];
          initialCluster = ["etcd1=https://etcd1:2380" "etcd2=https://etcd2:2380" "etcd3=https://etcd3:2380"];
          initialAdvertisePeerUrls = ["https://etcd3:2380"];
        };
      };

      kubeMaster1 = { config, pkgs, lib, nodes, ... }: {
        require = [kubeMasterConfig];
      };

      kubeMaster2 = { config, pkgs, lib, nodes, ... }: {
        require = [kubeMasterConfig];
      };

      # Kubernetes TCP load balancer
      kubernetes = { config, pkgs, ... }: {
        # kubernetes
        networking.firewall.allowedTCPPorts = [ 443 ];

        services.haproxy.enable = true;
        services.haproxy.config = ''
          global
              log 127.0.0.1 local0 notice
              user haproxy
              group haproxy

          defaults
              log global
              retries 2
              timeout connect 3000
              timeout server 5000
              timeout client 5000

          listen kubernetes
              bind 0.0.0.0:443
              mode tcp
              option ssl-hello-chk
              balance roundrobin
              server kube-master-1 kubeMaster1:443 check
              server kube-master-2 kubeMaster2:443 check
        '';
      };

      kubeWorker1 = { config, pkgs, lib, nodes, ... }: {
        require = [kubeWorkerConfig];
      };

      kubeWorker2 = { config, pkgs, lib, nodes, ... }: {
        require = [kubeWorkerConfig];
      };
    };

    testScript = ''
      startAll;

      ${testSimplePod}
    '';
  };
}