kubernetes: update to 0.12.1, fix module

nixos/modules/services/cluster/kubernetes.nix

@@ -44,6 +44,12 @@ in {
       type = types.path;
     };
 
+    dockerCfg = mkOption {
+      description = "Kubernetes contents of dockercfg file.";
+      default = "";
+      type = types.lines;
+    };
+
     apiserver = {
       enable = mkOption {
         description = "Whether to enable kubernetes apiserver.";
@@ -217,13 +223,13 @@ in {
       };
 
       machines = mkOption {
-        description = "Kubernetes apiserver list of machines to schedule to schedule onto";
+        description = "Kubernetes controller list of machines to schedule to schedule onto";
         default = [];
         type = types.listOf types.str;
       };
 
       extraOpts = mkOption {
-        description = "Kubernetes scheduler extra command line options.";
+        description = "Kubernetes controller extra command line options.";
         default = "";
         type = types.str;
       };
@@ -260,6 +266,30 @@ in {
         type = types.bool;
       };
 
+      apiServers = mkOption {
+        description = "Kubernetes kubelet list of Kubernetes API servers for publishing events, and reading pods and services.";
+        default = ["${cfg.apiserver.address}:${toString cfg.apiserver.port}"];
+        type = types.listOf types.str;
+      };
+
+      cadvisorPort = mkOption {
+        description = "Kubernetes kubelet local cadvisor port.";
+        default = config.services.cadvisor.port;
+        type = types.int;
+      };
+
+      clusterDns = mkOption {
+        description = "Use alternative dns.";
+        default = "";
+        type = types.str;
+      };
+
+      clusterDomain = mkOption {
+        description = "Use alternative domain.";
+        default = "";
+        type = types.str;
+      };
+
       extraOpts = mkOption {
         description = "Kubernetes kubelet extra command line options.";
         default = "";
@@ -295,6 +325,7 @@ in {
       systemd.services.kubernetes-apiserver = {
         description = "Kubernetes Api Server";
         wantedBy = [ "multi-user.target" ];
+        requires = ["kubernetes-setup.service"];
         after = [ "network-interfaces.target" "etcd.service" ];
         serviceConfig = {
           ExecStart = let
@@ -306,26 +337,25 @@ in {
                 (concatImapStringsSep "\n" (i: v: v + "," + (toString i))
                     (mapAttrsToList (name: token: token + "," + name) cfg.apiserver.tokenAuth));
           in ''${cfg.package}/bin/kube-apiserver \
-            -etcd_servers=${concatMapStringsSep "," (f: "http://${f}") cfg.etcdServers} \
+            --etcd_servers=${concatMapStringsSep "," (f: "http://${f}") cfg.etcdServers} \
-            -address=${cfg.apiserver.address} \
+            --address=${cfg.apiserver.address} \
-            -port=${toString cfg.apiserver.port} \
+            --port=${toString cfg.apiserver.port} \
-            -read_only_port=${toString cfg.apiserver.readOnlyPort} \
+            --read_only_port=${toString cfg.apiserver.readOnlyPort} \
-            -public_address_override=${cfg.apiserver.publicAddress} \
+            --public_address_override=${cfg.apiserver.publicAddress} \
-            -allow_privileged=${if cfg.apiserver.allowPrivileged then "true" else "false"} \
+            --allow_privileged=${if cfg.apiserver.allowPrivileged then "true" else "false"} \
             ${optionalString (cfg.apiserver.tlsCertFile!="")
-              "-tls_cert_file=${cfg.apiserver.tlsCertFile}"} \
+              "--tls_cert_file=${cfg.apiserver.tlsCertFile}"} \
             ${optionalString (cfg.apiserver.tlsPrivateKeyFile!="")
-              "-tls_private_key_file=${cfg.apiserver.tlsPrivateKeyFile}"} \
+              "--tls_private_key_file=${cfg.apiserver.tlsPrivateKeyFile}"} \
             ${optionalString (cfg.apiserver.tokenAuth!=[])
-              "-token_auth_file=${tokenAuthFile}"} \
+              "--token_auth_file=${tokenAuthFile}"} \
-            -authorization_mode=${cfg.apiserver.authorizationMode} \
+            --authorization_mode=${cfg.apiserver.authorizationMode} \
             ${optionalString (cfg.apiserver.authorizationMode == "ABAC")
-              "-authorization_policy_file=${authorizationPolicyFile}"} \
+              "--authorization_policy_file=${authorizationPolicyFile}"} \
-            ${optionalString (cfg.apiserver.tlsCertFile!="" && cfg.apiserver.tlsCertFile!="")
+            --secure_port=${toString cfg.apiserver.securePort} \
-              "-secure_port=${toString cfg.apiserver.securePort}"} \
+            --portal_net=${cfg.apiserver.portalNet} \
-            -portal_net=${cfg.apiserver.portalNet} \
+            --logtostderr=true \
-            -logtostderr=true \
+            ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \
-            ${optionalString cfg.verbose "-v=6 -log_flush_frequency=1s"} \
             ${cfg.apiserver.extraOpts}
           '';
           User = "kubernetes";
@@ -345,11 +375,11 @@ in {
         after = [ "network-interfaces.target" "kubernetes-apiserver.service" ];
         serviceConfig = {
           ExecStart = ''${cfg.package}/bin/kube-scheduler \
-            -address=${cfg.scheduler.address} \
+            --address=${cfg.scheduler.address} \
-            -port=${toString cfg.scheduler.port} \
+            --port=${toString cfg.scheduler.port} \
-            -master=${cfg.scheduler.master} \
+            --master=${cfg.scheduler.master} \
-            -logtostderr=true \
+            --logtostderr=true \
-            ${optionalString cfg.verbose "-v=6 -log_flush_frequency=1s"} \
+            ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \
             ${cfg.scheduler.extraOpts}
           '';
           User = "kubernetes";
@@ -364,13 +394,12 @@ in {
         after = [ "network-interfaces.target" "kubernetes-apiserver.service" ];
         serviceConfig = {
           ExecStart = ''${cfg.package}/bin/kube-controller-manager \
-            -address=${cfg.controllerManager.address} \
+            --address=${cfg.controllerManager.address} \
-            -port=${toString cfg.controllerManager.port} \
+            --port=${toString cfg.controllerManager.port} \
-            -master=${cfg.controllerManager.master} \
+            --master=${cfg.controllerManager.master} \
-            ${optionalString (cfg.controllerManager.machines != [])
+            --machines=${concatStringsSep "," cfg.controllerManager.machines} \
-                "-machines=${concatStringsSep "," cfg.controllerManager.machines}"} \
+            --logtostderr=true \
-            -logtostderr=true \
+            ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \
-            ${optionalString cfg.verbose "-v=6 -log_flush_frequency=1s"} \
             ${cfg.controllerManager.extraOpts}
           '';
           User = "kubernetes";
@@ -382,23 +411,28 @@ in {
       systemd.services.kubernetes-kubelet = {
         description = "Kubernetes Kubelet Service";
         wantedBy = [ "multi-user.target" ];
+        requires = ["kubernetes-setup.service"];
         after = [ "network-interfaces.target" "etcd.service" "docker.service" ];
-        serviceConfig = {
+        script = ''
-          ExecStart = ''${cfg.package}/bin/kubelet \
+          export PATH="/bin:/sbin:/usr/bin:/usr/sbin:$PATH"
-            -etcd_servers=${concatMapStringsSep "," (f: "http://${f}") cfg.etcdServers} \
+          exec ${cfg.package}/bin/kubelet \
-            -address=${cfg.kubelet.address} \
+            --etcd_servers=${concatMapStringsSep "," (f: "http://${f}") cfg.etcdServers} \
-            -port=${toString cfg.kubelet.port} \
+            --api_servers=${concatMapStringsSep "," (f: "http://${f}") cfg.kubelet.apiServers}  \
-            -hostname_override=${cfg.kubelet.hostname} \
+            --address=${cfg.kubelet.address} \
-            -allow_privileged=${if cfg.kubelet.allowPrivileged then "true" else "false"} \
+            --port=${toString cfg.kubelet.port} \
-            -root_dir=${cfg.dataDir} \
+            --hostname_override=${cfg.kubelet.hostname} \
-            -logtostderr=true \
+            --allow_privileged=${if cfg.kubelet.allowPrivileged then "true" else "false"} \
-            ${optionalString cfg.verbose "-v=6 -log_flush_frequency=1s"} \
+            --root_dir=${cfg.dataDir} \
+            --cadvisor_port=${toString cfg.kubelet.cadvisorPort} \
+            ${optionalString (cfg.kubelet.clusterDns != "")
+                ''--cluster_dns=${cfg.kubelet.clusterDns}''} \
+            ${optionalString (cfg.kubelet.clusterDomain != "")
+                ''--cluster_domain=${cfg.kubelet.clusterDomain}''} \
+            --logtostderr=true \
+            ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \
             ${cfg.kubelet.extraOpts}
           '';
-          User = "kubernetes";
+        serviceConfig.WorkingDirectory = cfg.dataDir;
-          PermissionsStartOnly = true;
-          WorkingDirectory = cfg.dataDir;
-        };
       };
     })
 
@@ -409,10 +443,10 @@ in {
         after = [ "network-interfaces.target" "etcd.service" ];
         serviceConfig = {
           ExecStart = ''${cfg.package}/bin/kube-proxy \
-            -etcd_servers=${concatMapStringsSep "," (s: "http://${s}") cfg.etcdServers} \
+            --etcd_servers=${concatMapStringsSep "," (s: "http://${s}") cfg.etcdServers} \
-            -bind_address=${cfg.proxy.address} \
+            --bind_address=${cfg.proxy.address} \
-            -logtostderr=true \
+            --logtostderr=true \
-            ${optionalString cfg.verbose "-v=6 -log_flush_frequency=1s"} \
+            ${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \
             ${cfg.proxy.extraOpts}
           '';
         };
@@ -427,6 +461,8 @@ in {
 
     (mkIf (any (el: el == "node") cfg.roles) {
       virtualisation.docker.enable = mkDefault true;
+      services.cadvisor.enable = mkDefault true;
+      services.cadvisor.port = mkDefault 4194;
       services.kubernetes.kubelet.enable = mkDefault true;
       services.kubernetes.proxy.enable = mkDefault true;
     })
@@ -442,6 +478,16 @@ in {
         cfg.kubelet.enable ||
         cfg.proxy.enable
     ) {
+      systemd.services.kubernetes-setup = {
+        description = "Kubernetes setup.";
+        serviceConfig.Type = "oneshot";
+        script = ''
+          mkdir -p /var/run/kubernetes
+          chown kubernetes /var/run/kubernetes
+          ln -fs ${pkgs.writeText "kubernetes-dockercfg" cfg.dockerCfg} /var/run/kubernetes/.dockercfg
+        '';
+      };
+
       services.kubernetes.package = mkDefault pkgs.kubernetes;
 
       environment.systemPackages = [ cfg.package ];

nixos/tests/kubernetes.nix

@@ -13,7 +13,7 @@ import ./make-test.nix rec {
           id: redis-master-pod
           containers:
             - name: master
-              image: master:5000/scratch
+              image: master:5000/nix
               cpu: 100
               ports:
                 - name: redis-server
@@ -50,8 +50,8 @@ import ./make-test.nix rec {
           virtualisation.memorySize = 768;
           services.kubernetes = {
             roles = ["master" "node"];
+            dockerCfg = ''{"master:5000":{}}'';
             controllerManager.machines = ["master" "node"];
-            kubelet.extraOpts = "-network_container_image=master:5000/pause";
             apiserver.address = "0.0.0.0";
             verbose = true;
           };
@@ -94,7 +94,8 @@ import ./make-test.nix rec {
         {
           services.kubernetes = {
             roles = ["node"];
-            kubelet.extraOpts = "-network_container_image=master:5000/pause";
+            dockerCfg = ''{"master:5000":{}}'';
+            kubelet.apiServers = ["master:8080"];
             verbose = true;
           };
           virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false -b cbr0 --insecure-registry master:5000";
@@ -155,14 +156,14 @@ import ./make-test.nix rec {
     $node->waitForUnit("kubernetes-kubelet.service");
     $node->waitForUnit("kubernetes-proxy.service");
 
-    $master->waitUntilSucceeds("kubecfg list minions | grep master");
+    $master->waitUntilSucceeds("kubectl get minions | grep master");
-    $master->waitUntilSucceeds("kubecfg list minions | grep node");
+    $master->waitUntilSucceeds("kubectl get minions | grep node");
 
     $client->waitForUnit("docker.service");
-    $client->succeed("tar cv --files-from /dev/null | docker import - scratch");
+    $client->succeed("tar cv --files-from /dev/null | docker import - nix");
-    $client->succeed("docker tag scratch master:5000/scratch");
+    $client->succeed("docker tag nix master:5000/nix");
     $master->waitForUnit("docker-registry.service");
-    $client->succeed("docker push master:5000/scratch");
+    $client->succeed("docker push master:5000/nix");
     $client->succeed("mkdir -p /root/pause");
     $client->succeed("cp /etc/test/pause /root/pause/");
     $client->succeed("cp /etc/test/Dockerfile /root/pause/");

pkgs/applications/networking/cluster/kubernetes/default.nix

@@ -1,14 +1,14 @@
-{ stdenv, fetchFromGitHub, which, go, makeWrapper, iptables,rsync }:
+{ stdenv, fetchFromGitHub, which, go, makeWrapper, iptables, rsync, utillinux, coreutils }:
 
 stdenv.mkDerivation rec {
   name = "kubernetes-${version}";
-  version = "v0.5.4";
+  version = "0.12.1";
 
   src = fetchFromGitHub {
     owner = "GoogleCloudPlatform";
     repo = "kubernetes";
-    rev = version;
+    rev = "v${version}";
-    sha256 = "1pipcqpjz9zsi4kfsbdvbbbia642l4xg50pznjw5v061c5xk7vnk";
+    sha256 = "1891wpssfp04nkk1h4y3cdgn096b0kq16pc0m2fzilbh3daa6pml";
   };
 
   buildInputs = [ makeWrapper which go iptables rsync ];
@@ -24,7 +24,8 @@ stdenv.mkDerivation rec {
   '';
 
   preFixup = ''
-    wrapProgram "$out/bin/kube-proxy" --set "PATH" "${iptables}/bin"
+    wrapProgram "$out/bin/kube-proxy" --prefix PATH : "${iptables}/bin"
+    wrapProgram "$out/bin/kubelet" --prefix PATH : "${utillinux}/bin"
   '';
 
   meta = with stdenv.lib; {