Add expiration to bootstrap tokens (#823)

* Add expiration to bootstrap tokens, increase node password complexity * Revert node password complexity * Token validity over api to one hour
sapcc · Sep 5, 2023 · a9473f3 · a9473f3
1 parent c4fea6a
commit a9473f3
Show file tree

Hide file tree

Showing 9 changed files with 28 additions and 27 deletions.
diff --git a/charts/kube-master/templates/_token.csv.tpl b/charts/kube-master/templates/_token.csv.tpl
diff --git a/charts/kube-master/templates/api.yaml b/charts/kube-master/templates/api.yaml
@@ -97,12 +97,6 @@ spec:
               - key: openstack.config
                 path: openstack.config
         {{- end }}
-        - name: bootstrap
-          secret:
-            secretName: {{ include "master.fullname" . }}-generated
-            items:
-              - key: token.csv
-                path: token.csv
         - name: config
           configMap:
             name: {{ include "master.fullname" . }}
@@ -206,7 +200,6 @@ spec:
             - --experimental-bootstrap-token-auth=true
             - --runtime-config=rbac.authorization.k8s.io/v1alpha1,extensions/v1beta1=true,extensions/v1beta1/thirdpartyresources=true
 {{- end }}
-            - --token-auth-file=/etc/kubernetes/bootstrap/token.csv
             - --service-cluster-ip-range={{ .Values.serviceCIDR }}
             - --kubelet-preferred-address-types=InternalIP
 {{- if (semverCompare ">= 1.10-0" .Values.version.kubernetes) }}
@@ -291,9 +284,6 @@ spec:
               name: cloudprovider
               readOnly: true
             {{- end }}
-            - mountPath: /etc/kubernetes/bootstrap
-              name: bootstrap
-              readOnly: true
 {{- if and (.Values.etcd.backup.enabled) (semverCompare "< 1.19" .Values.version.kubernetes) }}
             - mountPath: /liveness-probe
               name: liveness-probe

diff --git a/charts/kube-master/templates/secrets.yaml b/charts/kube-master/templates/secrets.yaml
@@ -17,4 +17,3 @@ data:
   openstack-csi.config: {{ include (print $.Template.BasePath "/_openstack-csi.config.tpl") . | b64enc}}
   {{- end }}
   {{- end }}
-  token.csv: {{ include (print $.Template.BasePath "/_token.csv.tpl") . | b64enc }}
diff --git a/pkg/api/handlers/get_cluster_bootstrap.go b/pkg/api/handlers/get_cluster_bootstrap.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"strings"
 	"text/template"
+	"time"
 
 	"github.com/ghodss/yaml"
 	"github.com/go-openapi/runtime/middleware"
@@ -79,7 +80,7 @@ func (d *getBootstrapConfig) Handle(params operations.GetBootstrapConfigParams,
 		return NewErrorResponse(&operations.GetBootstrapConfigDefault{}, 500, "Failed to create cluster client: %s", err)
 	}
 
-	token, tokenSecret, err := bootstraptoken.GenerateBootstrapToken()
+	token, tokenSecret, err := bootstraptoken.GenerateBootstrapToken(1 * time.Hour)
 	if err != nil {
 		return NewErrorResponse(&operations.GetBootstrapConfigDefault{}, 500, "Failed to generate bootstrap token: %s", err)
 	}

diff --git a/pkg/controller/ground.go b/pkg/controller/ground.go
@@ -631,7 +631,6 @@ func (op *GroundControl) createKluster(kluster *v1.Kluster) error {
 	if _, err := certFactory.Ensure(); err != nil {
 		return fmt.Errorf("Failed to generate certificates: %s", err)
 	}
-	klusterSecret.BootstrapToken = util.GenerateBootstrapToken()
 
 	if !kluster.Spec.NoCloud {
 		adminClient, err := op.Factories.Openstack.AdminClient()

diff --git a/pkg/controller/launch/pool_manager.go b/pkg/controller/launch/pool_manager.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/go-kit/log"
 	core_v1 "k8s.io/api/core/v1"
@@ -20,6 +21,7 @@ import (
 	kubernikus_listers "github.com/sapcc/kubernikus/pkg/generated/listers/kubernikus/v1"
 	"github.com/sapcc/kubernikus/pkg/templates"
 	"github.com/sapcc/kubernikus/pkg/util"
+	"github.com/sapcc/kubernikus/pkg/util/bootstraptoken"
 	"github.com/sapcc/kubernikus/pkg/util/generator"
 	"github.com/sapcc/kubernikus/pkg/version"
 )
@@ -200,14 +202,26 @@ func (cpm *ConcretePoolManager) CreateNode() (id string, err error) {
 
 	nodeName := generator.SimpleNameGenerator.GenerateName(fmt.Sprintf(util.NODE_NAMING_PATTERN_PREFIX, cpm.Kluster.Spec.Name, cpm.Pool.Name))
 
+	client, err := cpm.Clients.Satellites.ClientFor(cpm.Kluster)
+	if err != nil {
+		return "", fmt.Errorf("Couldn't get client for kluster: %s", err)
+	}
+
 	calicoNetworking := false
-	if client, err := cpm.Clients.Satellites.ClientFor(cpm.Kluster); err == nil {
-		if _, err := client.AppsV1().DaemonSets("kube-system").Get(context.TODO(), "calico-node", metav1.GetOptions{}); err == nil {
-			calicoNetworking = true
-		}
+	if _, err := client.AppsV1().DaemonSets("kube-system").Get(context.TODO(), "calico-node", metav1.GetOptions{}); err == nil {
+		calicoNetworking = true
+	}
+
+	token, tokenSecret, err := bootstraptoken.GenerateBootstrapToken(30 * time.Minute)
+	if err != nil {
+		return "", fmt.Errorf("Node bootstrap token generation failed: %s", err)
+	}
+
+	if _, err := client.CoreV1().Secrets(tokenSecret.Namespace).Create(context.TODO(), tokenSecret, metav1.CreateOptions{}); err != nil {
+		return "", fmt.Errorf("Node bootstrap token secret creation failed: %s", err)
 	}
 
-	userdata, err := templates.Ignition.GenerateNode(cpm.Kluster, cpm.Pool, nodeName, secret, calicoNetworking, cpm.imageRegistry, cpm.Logger)
+	userdata, err := templates.Ignition.GenerateNode(cpm.Kluster, cpm.Pool, nodeName, token, secret, calicoNetworking, cpm.imageRegistry, cpm.Logger)
 	if err != nil {
 		return "", err
 	}

diff --git a/pkg/templates/ignition.go b/pkg/templates/ignition.go
@@ -72,7 +72,7 @@ func (i *ignition) getIgnitionTemplate(kluster *kubernikusv1.Kluster) (string, e
 	}
 }
 
-func (i *ignition) GenerateNode(kluster *kubernikusv1.Kluster, pool *models.NodePool, nodeName string, secret *kubernikusv1.Secret, calicoNetworking bool, imageRegistry version.ImageRegistry, logger log.Logger) ([]byte, error) {
+func (i *ignition) GenerateNode(kluster *kubernikusv1.Kluster, pool *models.NodePool, nodeName, token string, secret *kubernikusv1.Secret, calicoNetworking bool, imageRegistry version.ImageRegistry, logger log.Logger) ([]byte, error) {
 
 	ignition, err := i.getIgnitionTemplate(kluster)
 	if err != nil {
@@ -175,7 +175,7 @@ func (i *ignition) GenerateNode(kluster *kubernikusv1.Kluster, pool *models.Node
 		KubeletClientsCA:                   secret.KubeletClientsCACertificate,
 		ApiserverClientsSystemKubeProxy:    secret.ApiserverClientsKubeProxyCertificate,
 		ApiserverClientsSystemKubeProxyKey: secret.ApiserverClientsKubeProxyPrivateKey,
-		BootstrapToken:                     secret.BootstrapToken,
+		BootstrapToken:                     token,
 		ClusterCIDR:                        kluster.ClusterCIDR(),
 		ClusterDNSAddress:                  kluster.Spec.DNSAddress,
 		ClusterDomain:                      kluster.Spec.DNSDomain,

diff --git a/pkg/templates/ignition_test.go b/pkg/templates/ignition_test.go
@@ -96,7 +96,7 @@ func TestGenerateNode(t *testing.T) {
 
 	for version := range imageRegistry.Versions {
 		kluster.Spec.Version = version
-		data, err := Ignition.GenerateNode(kluster, nil, "test", &testKlusterSecret, false, imageRegistry, log.NewNopLogger())
+		data, err := Ignition.GenerateNode(kluster, nil, "test", "abc123", &testKlusterSecret, false, imageRegistry, log.NewNopLogger())
 
 		if assert.NoError(t, err, "Failed to generate node for version %s", version) {
 			//Ensure we rendered the expected template
@@ -116,14 +116,14 @@ func TestNodeLabels(t *testing.T) {
 
 	pool := &models.NodePool{Name: "some-name"}
 
-	data, err := Ignition.GenerateNode(kluster, pool, "test", &testKlusterSecret, false, imageRegistry, log.NewNopLogger())
+	data, err := Ignition.GenerateNode(kluster, pool, "test", "abc123", &testKlusterSecret, false, imageRegistry, log.NewNopLogger())
 	if assert.NoError(t, err, "Failed to generate node") {
 		//Ensure we rendered the expected template
 		assert.Contains(t, string(data), fmt.Sprintf("--node-labels=ccloud.sap.com/nodepool=%s", pool.Name))
 	}
 
 	gpuPool := &models.NodePool{Name: "some-name", Flavor: "zghuh"}
-	data, err = Ignition.GenerateNode(kluster, gpuPool, "test", &testKlusterSecret, false, imageRegistry, log.NewNopLogger())
+	data, err = Ignition.GenerateNode(kluster, gpuPool, "test", "abc123", &testKlusterSecret, false, imageRegistry, log.NewNopLogger())
 	if assert.NoError(t, err, "Failed to generate node") {
 		//Ensure we rendered the expected template
 		assert.Contains(t, string(data), fmt.Sprintf("--node-labels=ccloud.sap.com/nodepool=%s,gpu=nvidia-tesla-v100", pool.Name))

diff --git a/pkg/util/bootstraptoken/util.go b/pkg/util/bootstraptoken/util.go
@@ -10,7 +10,7 @@ import (
 	bootstraptokenutil "k8s.io/cluster-bootstrap/token/util"
 )
 
-func GenerateBootstrapToken() (string, *corev1.Secret, error) {
+func GenerateBootstrapToken(t time.Duration) (string, *corev1.Secret, error) {
 
 	token, err := bootstraptokenutil.GenerateBootstrapToken()
 	if err != nil {
@@ -34,7 +34,7 @@ func GenerateBootstrapToken() (string, *corev1.Secret, error) {
 			bootstraptokenapi.BootstrapTokenDescriptionKey:      "Bootstrap token generated by Kubernikus",
 			bootstraptokenapi.BootstrapTokenIDKey:               tokenID,
 			bootstraptokenapi.BootstrapTokenSecretKey:           tokenSecret,
-			bootstraptokenapi.BootstrapTokenExpirationKey:       metav1.Now().Add(24 * time.Hour).Format(time.RFC3339),
+			bootstraptokenapi.BootstrapTokenExpirationKey:       metav1.Now().Add(t).Format(time.RFC3339),
 			bootstraptokenapi.BootstrapTokenUsageAuthentication: "true",
 			bootstraptokenapi.BootstrapTokenUsageSigningKey:     "true",
 		},