NixOS · Mar 3, 2019 · Mar 3, 2019 · Mar 3, 2019 · Mar 3, 2019 · Mar 3, 2019
diff --git a/lib/fixed-points.nix b/lib/fixed-points.nix
@@ -30,9 +30,12 @@ rec {
   #     nix-repl> converge (x: x / 2) 16
   #     0
   converge = f: x:
-    if (f x) == x
-    then x
-    else converge f (f x);
+    let
+      x' = f x;
+    in
+      if x' == x
+      then x
+      else converge f x';
 
   # Modify the contents of an explicitly recursive attribute set in a way that
   # honors `self`-references. This is accomplished with a function

diff --git a/nixos/modules/services/cluster/kubernetes/addon-manager.nix b/nixos/modules/services/cluster/kubernetes/addon-manager.nix
@@ -63,18 +63,49 @@ in
     };
 
     enable = mkEnableOption "Whether to enable Kubernetes addon manager.";
+
+    kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes addon manager";
+    bootstrapAddonsKubeconfig = top.lib.mkKubeConfigOptions "Kubernetes addon manager bootstrap";
   };
 
   ###### implementation
-  config = mkIf cfg.enable {
+  config = let
+
+    addonManagerPaths = filter (a: a != null) [
+      cfg.kubeconfig.caFile
+      cfg.kubeconfig.certFile
+      cfg.kubeconfig.keyFile
+    ];
+    bootstrapAddonsPaths = filter (a: a != null) [
+      cfg.bootstrapAddonsKubeconfig.caFile
+      cfg.bootstrapAddonsKubeconfig.certFile
+      cfg.bootstrapAddonsKubeconfig.keyFile
+    ];
+
+  in mkIf cfg.enable {
     environment.etc."kubernetes/addons".source = "${addons}/";
 
+    #TODO: Get rid of kube-addon-manager in the future for the following reasons
+    # - it is basically just a shell script wrapped around kubectl
+    # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount
+    # - it is designed to be used with k8s system components only
+    # - it would be better with a more Nix-oriented way of managing addons
     systemd.services.kube-addon-manager = {
       description = "Kubernetes addon manager";
       wantedBy = [ "kubernetes.target" ];
-      after = [ "kube-apiserver.service" ];
-      environment.ADDON_PATH = "/etc/kubernetes/addons/";
-      path = [ pkgs.gawk ];
+      after = [ "kube-node-online.target" ];
+      before = [ "kubernetes.target" ];
+      environment = {
+        ADDON_PATH = "/etc/kubernetes/addons/";
+        KUBECONFIG = top.lib.mkKubeConfig "kube-addon-manager" cfg.kubeconfig;
+      };
+      path = with pkgs; [ gawk kubectl ];
+      preStart = ''
+        until kubectl -n kube-system get serviceaccounts/default 2>/dev/null; do
+          echo kubectl -n kube-system get serviceaccounts/default: exit status $?
+          sleep 2
+        done
+      '';
       serviceConfig = {
         Slice = "kubernetes.slice";
         ExecStart = "${top.package}/bin/kube-addons";
@@ -84,8 +115,52 @@ in
         Restart = "on-failure";
         RestartSec = 10;
       };
+      unitConfig.ConditionPathExists = addonManagerPaths;
     };
 
+    systemd.paths.kube-addon-manager = {
+      wantedBy = [ "kube-addon-manager.service" ];
+      pathConfig = {
+        PathExists = addonManagerPaths;
+        PathChanged = addonManagerPaths;
+      };
+    };
+
+    services.kubernetes.addonManager.kubeconfig.server = mkDefault top.apiserverAddress;
+
+    systemd.services.kube-addon-manager-bootstrap = mkIf (top.apiserver.enable && top.addonManager.bootstrapAddons != {}) {
+      wantedBy = [ "kube-control-plane-online.target" ];
+      after = [ "kube-apiserver.service" ];
+      before = [ "kube-control-plane-online.target" ];
+      path = [ pkgs.kubectl ];
+      environment = {
+        KUBECONFIG = top.lib.mkKubeConfig "kube-addon-manager-bootstrap" cfg.bootstrapAddonsKubeconfig;
+      };
+      preStart = with pkgs; let
+        files = mapAttrsToList (n: v: writeText "${n}.json" (builtins.toJSON v))
+          cfg.bootstrapAddons;
+      in ''
+        until kubectl auth can-i '*' '*' -q 2>/dev/null; do
+          echo kubectl auth can-i '*' '*': exit status $?
+          sleep 2
+        done
+
+        kubectl apply -f ${concatStringsSep " \\\n -f " files}
+      '';
+      script = "echo Ok";
+      unitConfig.ConditionPathExists = bootstrapAddonsPaths;
+    };
+
+    systemd.paths.kube-addon-manager-bootstrap = {
+      wantedBy = [ "kube-addon-manager-bootstrap.service" ];
+      pathConfig = {
+        PathExists = bootstrapAddonsPaths;
+        PathChanged = bootstrapAddonsPaths;
+      };
+    };
+
+    services.kubernetes.addonManager.bootstrapAddonsKubeconfig.server = mkDefault top.apiserverAddress;
+
     services.kubernetes.addonManager.bootstrapAddons = mkIf isRBACEnabled
     (let
       name = system:kube-addon-manager;

diff --git a/nixos/modules/services/cluster/kubernetes/addons/dashboard.nix b/nixos/modules/services/cluster/kubernetes/addons/dashboard.nix
@@ -169,6 +169,23 @@ in {
         };
       };
 
+      kubernetes-dashboard-cm = {
+        apiVersion = "v1";
+        kind = "ConfigMap";
+        metadata = {
+          labels = {
+            k8s-app = "kubernetes-dashboard";
+            # Allows editing resource and makes sure it is created first.
+            "addonmanager.kubernetes.io/mode" = "EnsureExists";
+          };
+          name = "kubernetes-dashboard-settings";
+          namespace = "kube-system";
+        };
+      };
+    };
+
+    services.kubernetes.addonManager.bootstrapAddons = mkMerge [{
+
       kubernetes-dashboard-sa = {
         apiVersion = "v1";
         kind = "ServiceAccount";
@@ -210,20 +227,9 @@ in {
         };
         type = "Opaque";
       };
-      kubernetes-dashboard-cm = {
-        apiVersion = "v1";
-        kind = "ConfigMap";
-        metadata = {
-          labels = {
-            k8s-app = "kubernetes-dashboard";
-            # Allows editing resource and makes sure it is created first.
-            "addonmanager.kubernetes.io/mode" = "EnsureExists";
-          };
-          name = "kubernetes-dashboard-settings";
-          namespace = "kube-system";
-        };
-      };
-    } // (optionalAttrs cfg.rbac.enable
+    }
+
+    (optionalAttrs cfg.rbac.enable
       (let
         subjects = [{
           kind = "ServiceAccount";
@@ -323,6 +329,6 @@ in {
             inherit subjects;
           };
         })
-    ));
+    ))];
   };
 }
diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix
@@ -184,6 +184,18 @@ in
       type = bool;
     };
 
+    proxyClientCertFile = mkOption {
+      description = "Client certificate to use for connections to proxy.";
+      default = null;
+      type = nullOr path;
+    };
+
+    proxyClientKeyFile = mkOption {
+      description = "Key to use for connections to proxy.";
+      default = null;
+      type = nullOr path;
+    };
+
     runtimeConfig = mkOption {
       description = ''
         Api runtime configuration. See
@@ -272,11 +284,32 @@ in
   ###### implementation
   config = mkMerge [
 
-    (mkIf cfg.enable {
+    (let
+
+      apiserverPaths = filter (a: a != null) [
+        cfg.clientCaFile
+        cfg.etcd.caFile
+        cfg.etcd.certFile
+        cfg.etcd.keyFile
+        cfg.kubeletClientCaFile
+        cfg.kubeletClientCertFile
+        cfg.kubeletClientKeyFile
+        cfg.serviceAccountKeyFile
+        cfg.tlsCertFile
+        cfg.tlsKeyFile
+      ];
+      etcdPaths = filter (a: a != null) [
+        config.services.etcd.trustedCaFile
+        config.services.etcd.certFile
+        config.services.etcd.keyFile
+      ];
+
+    in mkIf cfg.enable {
         systemd.services.kube-apiserver = {
           description = "Kubernetes APIServer Service";
-          wantedBy = [ "kubernetes.target" ];
-          after = [ "network.target" ];
+          wantedBy = [ "kube-control-plane-online.target" ];
+          after = [ "certmgr.service" ];
+          before = [ "kube-control-plane-online.target" ];
           serviceConfig = {
             Slice = "kubernetes.slice";
             ExecStart = ''${top.package}/bin/kube-apiserver \
@@ -316,6 +349,10 @@ in
                 "--kubelet-client-certificate=${cfg.kubeletClientCertFile}"} \
               ${optionalString (cfg.kubeletClientKeyFile != null)
                 "--kubelet-client-key=${cfg.kubeletClientKeyFile}"} \
+              ${optionalString (cfg.proxyClientCertFile != null)
+                "--proxy-client-cert-file=${cfg.proxyClientCertFile}"} \
+              ${optionalString (cfg.proxyClientKeyFile != null)
+                "--proxy-client-key-file=${cfg.proxyClientKeyFile}"} \
               --insecure-bind-address=${cfg.insecureBindAddress} \
               --insecure-port=${toString cfg.insecurePort} \
               ${optionalString (cfg.runtimeConfig != "")
@@ -341,6 +378,15 @@ in
             Restart = "on-failure";
             RestartSec = 5;
           };
+          unitConfig.ConditionPathExists = apiserverPaths;
+        };
+
+        systemd.paths.kube-apiserver = mkIf top.apiserver.enable {
+          wantedBy = [ "kube-apiserver.service" ];
+          pathConfig = {
+            PathExists = apiserverPaths;
+            PathChanged = apiserverPaths;
+          };
         };
 
         services.etcd = {
@@ -354,6 +400,18 @@ in
           initialAdvertisePeerUrls = mkDefault ["https://${top.masterAddress}:2380"];
         };
 
+        systemd.services.etcd = {
+          unitConfig.ConditionPathExists = etcdPaths;
+        };
+
+        systemd.paths.etcd = {
+          wantedBy = [ "etcd.service" ];
+          pathConfig = {
+            PathExists = etcdPaths;
+            PathChanged = etcdPaths;
+          };
+        };
+
         services.kubernetes.addonManager.bootstrapAddons = mkIf isRBACEnabled {
 
           apiserver-kubelet-api-admin-crb = {
@@ -389,6 +447,11 @@ in
                   ] ++ cfg.extraSANs;
           action = "systemctl restart kube-apiserver.service";
         };
+        apiserverProxyClient = mkCert {
+          name = "kube-apiserver-proxy-client";
+          CN = "front-proxy-client";
+          action = "systemctl restart kube-apiserver.service";
+        };
         apiserverKubeletClient = mkCert {
           name = "kube-apiserver-kubelet-client";
           CN = "system:kube-apiserver";

diff --git a/nixos/modules/services/cluster/kubernetes/controller-manager.nix b/nixos/modules/services/cluster/kubernetes/controller-manager.nix
@@ -104,11 +104,31 @@ in
   };
 
   ###### implementation
-  config = mkIf cfg.enable {
-    systemd.services.kube-controller-manager = {
+  config = let
+
+    controllerManagerPaths = filter (a: a != null) [
+      cfg.kubeconfig.caFile
+      cfg.kubeconfig.certFile
+      cfg.kubeconfig.keyFile
+      cfg.rootCaFile
+      cfg.serviceAccountKeyFile
+      cfg.tlsCertFile
+      cfg.tlsKeyFile
+    ];
+
+  in mkIf cfg.enable {
+    systemd.services.kube-controller-manager = rec {
       description = "Kubernetes Controller Manager Service";
-      wantedBy = [ "kubernetes.target" ];
+      wantedBy = [ "kube-control-plane-online.target" ];
       after = [ "kube-apiserver.service" ];
+      before = [ "kube-control-plane-online.target" ];
+      environment.KUBECONFIG = top.lib.mkKubeConfig "kube-controller-manager" cfg.kubeconfig;
+      preStart = ''
+        until kubectl auth can-i get /api -q 2>/dev/null; do
+          echo kubectl auth can-i get /api: exit status $?
+          sleep 2
+        done
+      '';
       serviceConfig = {
         RestartSec = "30s";
         Restart = "on-failure";
@@ -120,7 +140,7 @@ in
             "--cluster-cidr=${cfg.clusterCidr}"} \
           ${optionalString (cfg.featureGates != [])
             "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
-          --kubeconfig=${top.lib.mkKubeConfig "kube-controller-manager" cfg.kubeconfig} \
+          --kubeconfig=${environment.KUBECONFIG} \
           --leader-elect=${boolToString cfg.leaderElect} \
           ${optionalString (cfg.rootCaFile!=null)
             "--root-ca-file=${cfg.rootCaFile}"} \
@@ -141,7 +161,16 @@ in
         User = "kubernetes";
         Group = "kubernetes";
       };
-      path = top.path;
+      path = top.path ++ [ pkgs.kubectl ];
+      unitConfig.ConditionPathExists = controllerManagerPaths;
+    };
+
+    systemd.paths.kube-controller-manager = {
+      wantedBy = [ "kube-controller-manager.service" ];
+      pathConfig = {
+        PathExists = controllerManagerPaths;
+        PathChanged = controllerManagerPaths;
+      };
     };
 
     services.kubernetes.pki.certs = with top.lib; {

diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix
@@ -263,6 +263,30 @@ in {
         wantedBy = [ "multi-user.target" ];
       };
 
+      systemd.targets.kube-control-plane-online = {
+        wantedBy = [ "kubernetes.target" ];
+        before = [ "kubernetes.target" ];
+      };
+
+      systemd.services.kube-control-plane-online = rec {
+        description = "Kubernetes control plane is online";
+        wantedBy = [ "kube-control-plane-online.target" ];
+        after = [ "kube-scheduler.service" "kube-controller-manager.service" ];
+        before = [ "kube-control-plane-online.target" ];
+        environment.KUBECONFIG = cfg.lib.mkKubeConfig "default" cfg.kubeconfig;
+        path = [ pkgs.kubectl ];
+        preStart = ''
+          until kubectl get --raw=/healthz 2>/dev/null; do
+            echo kubectl get --raw=/healthz: exit status $?
+            sleep 3
+          done
+        '';
+        script = "echo Ok";
+        serviceConfig = {
+          TimeoutSec = "500";
+        };
+      };
+
       systemd.tmpfiles.rules = [
         "d /opt/cni/bin 0755 root root -"
         "d /run/kubernetes 0755 kubernetes kubernetes -"
@@ -286,6 +310,8 @@ in {
       services.kubernetes.apiserverAddress = mkDefault ("https://${if cfg.apiserver.advertiseAddress != null
                           then cfg.apiserver.advertiseAddress
                           else "${cfg.masterAddress}:${toString cfg.apiserver.securePort}"}");
+
+      services.kubernetes.kubeconfig.server = mkDefault cfg.apiserverAddress;
     })
   ];
 }