From a9473f379959af1a83889bc9c2a0a693a814fa54 Mon Sep 17 00:00:00 2001 From: Jan Knipper <9881823+jknipper@users.noreply.github.com> Date: Tue, 5 Sep 2023 15:25:09 +0200 Subject: [PATCH] Add expiration to bootstrap tokens (#823) * Add expiration to bootstrap tokens, increase node password complexity * Revert node password complexity * Token validity over api to one hour --- charts/kube-master/templates/_token.csv.tpl | 2 -- charts/kube-master/templates/api.yaml | 10 --------- charts/kube-master/templates/secrets.yaml | 1 - pkg/api/handlers/get_cluster_bootstrap.go | 3 ++- pkg/controller/ground.go | 1 - pkg/controller/launch/pool_manager.go | 24 ++++++++++++++++----- pkg/templates/ignition.go | 4 ++-- pkg/templates/ignition_test.go | 6 +++--- pkg/util/bootstraptoken/util.go | 4 ++-- 9 files changed, 28 insertions(+), 27 deletions(-) delete mode 100644 charts/kube-master/templates/_token.csv.tpl diff --git a/charts/kube-master/templates/_token.csv.tpl b/charts/kube-master/templates/_token.csv.tpl deleted file mode 100644 index 54deae543c..0000000000 --- a/charts/kube-master/templates/_token.csv.tpl +++ /dev/null @@ -1,2 +0,0 @@ -{{/* vim: set filetype=gotexttmpl: */ -}} -{{ required "missing bootstrapToken" .Values.bootstrapToken }},kubelet-bootstrap,10001,"system:bootstrappers" diff --git a/charts/kube-master/templates/api.yaml b/charts/kube-master/templates/api.yaml index df0a9200b0..1696c1b0be 100644 --- a/charts/kube-master/templates/api.yaml +++ b/charts/kube-master/templates/api.yaml @@ -97,12 +97,6 @@ spec: - key: openstack.config path: openstack.config {{- end }} - - name: bootstrap - secret: - secretName: {{ include "master.fullname" . }}-generated - items: - - key: token.csv - path: token.csv - name: config configMap: name: {{ include "master.fullname" . }} @@ -206,7 +200,6 @@ spec: - --experimental-bootstrap-token-auth=true - --runtime-config=rbac.authorization.k8s.io/v1alpha1,extensions/v1beta1=true,extensions/v1beta1/thirdpartyresources=true {{- end }} - - --token-auth-file=/etc/kubernetes/bootstrap/token.csv - --service-cluster-ip-range={{ .Values.serviceCIDR }} - --kubelet-preferred-address-types=InternalIP {{- if (semverCompare ">= 1.10-0" .Values.version.kubernetes) }} @@ -291,9 +284,6 @@ spec: name: cloudprovider readOnly: true {{- end }} - - mountPath: /etc/kubernetes/bootstrap - name: bootstrap - readOnly: true {{- if and (.Values.etcd.backup.enabled) (semverCompare "< 1.19" .Values.version.kubernetes) }} - mountPath: /liveness-probe name: liveness-probe diff --git a/charts/kube-master/templates/secrets.yaml b/charts/kube-master/templates/secrets.yaml index 4532ab3bf4..89eab28f5c 100644 --- a/charts/kube-master/templates/secrets.yaml +++ b/charts/kube-master/templates/secrets.yaml @@ -17,4 +17,3 @@ data: openstack-csi.config: {{ include (print $.Template.BasePath "/_openstack-csi.config.tpl") . | b64enc}} {{- end }} {{- end }} - token.csv: {{ include (print $.Template.BasePath "/_token.csv.tpl") . | b64enc }} diff --git a/pkg/api/handlers/get_cluster_bootstrap.go b/pkg/api/handlers/get_cluster_bootstrap.go index 0047dec438..4fa0dbbe11 100644 --- a/pkg/api/handlers/get_cluster_bootstrap.go +++ b/pkg/api/handlers/get_cluster_bootstrap.go @@ -4,6 +4,7 @@ import ( "context" "strings" "text/template" + "time" "github.com/ghodss/yaml" "github.com/go-openapi/runtime/middleware" @@ -79,7 +80,7 @@ func (d *getBootstrapConfig) Handle(params operations.GetBootstrapConfigParams, return NewErrorResponse(&operations.GetBootstrapConfigDefault{}, 500, "Failed to create cluster client: %s", err) } - token, tokenSecret, err := bootstraptoken.GenerateBootstrapToken() + token, tokenSecret, err := bootstraptoken.GenerateBootstrapToken(1 * time.Hour) if err != nil { return NewErrorResponse(&operations.GetBootstrapConfigDefault{}, 500, "Failed to generate bootstrap token: %s", err) } diff --git a/pkg/controller/ground.go b/pkg/controller/ground.go index 1fc28284a8..7dfa98efad 100644 --- a/pkg/controller/ground.go +++ b/pkg/controller/ground.go @@ -631,7 +631,6 @@ func (op *GroundControl) createKluster(kluster *v1.Kluster) error { if _, err := certFactory.Ensure(); err != nil { return fmt.Errorf("Failed to generate certificates: %s", err) } - klusterSecret.BootstrapToken = util.GenerateBootstrapToken() if !kluster.Spec.NoCloud { adminClient, err := op.Factories.Openstack.AdminClient() diff --git a/pkg/controller/launch/pool_manager.go b/pkg/controller/launch/pool_manager.go index 74baf23fcf..65b3e7ae70 100644 --- a/pkg/controller/launch/pool_manager.go +++ b/pkg/controller/launch/pool_manager.go @@ -5,6 +5,7 @@ import ( "fmt" "sort" "strings" + "time" "github.com/go-kit/log" core_v1 "k8s.io/api/core/v1" @@ -20,6 +21,7 @@ import ( kubernikus_listers "github.com/sapcc/kubernikus/pkg/generated/listers/kubernikus/v1" "github.com/sapcc/kubernikus/pkg/templates" "github.com/sapcc/kubernikus/pkg/util" + "github.com/sapcc/kubernikus/pkg/util/bootstraptoken" "github.com/sapcc/kubernikus/pkg/util/generator" "github.com/sapcc/kubernikus/pkg/version" ) @@ -200,14 +202,26 @@ func (cpm *ConcretePoolManager) CreateNode() (id string, err error) { nodeName := generator.SimpleNameGenerator.GenerateName(fmt.Sprintf(util.NODE_NAMING_PATTERN_PREFIX, cpm.Kluster.Spec.Name, cpm.Pool.Name)) + client, err := cpm.Clients.Satellites.ClientFor(cpm.Kluster) + if err != nil { + return "", fmt.Errorf("Couldn't get client for kluster: %s", err) + } + calicoNetworking := false - if client, err := cpm.Clients.Satellites.ClientFor(cpm.Kluster); err == nil { - if _, err := client.AppsV1().DaemonSets("kube-system").Get(context.TODO(), "calico-node", metav1.GetOptions{}); err == nil { - calicoNetworking = true - } + if _, err := client.AppsV1().DaemonSets("kube-system").Get(context.TODO(), "calico-node", metav1.GetOptions{}); err == nil { + calicoNetworking = true + } + + token, tokenSecret, err := bootstraptoken.GenerateBootstrapToken(30 * time.Minute) + if err != nil { + return "", fmt.Errorf("Node bootstrap token generation failed: %s", err) + } + + if _, err := client.CoreV1().Secrets(tokenSecret.Namespace).Create(context.TODO(), tokenSecret, metav1.CreateOptions{}); err != nil { + return "", fmt.Errorf("Node bootstrap token secret creation failed: %s", err) } - userdata, err := templates.Ignition.GenerateNode(cpm.Kluster, cpm.Pool, nodeName, secret, calicoNetworking, cpm.imageRegistry, cpm.Logger) + userdata, err := templates.Ignition.GenerateNode(cpm.Kluster, cpm.Pool, nodeName, token, secret, calicoNetworking, cpm.imageRegistry, cpm.Logger) if err != nil { return "", err } diff --git a/pkg/templates/ignition.go b/pkg/templates/ignition.go index 404aafe0d6..b884526bb1 100644 --- a/pkg/templates/ignition.go +++ b/pkg/templates/ignition.go @@ -72,7 +72,7 @@ func (i *ignition) getIgnitionTemplate(kluster *kubernikusv1.Kluster) (string, e } } -func (i *ignition) GenerateNode(kluster *kubernikusv1.Kluster, pool *models.NodePool, nodeName string, secret *kubernikusv1.Secret, calicoNetworking bool, imageRegistry version.ImageRegistry, logger log.Logger) ([]byte, error) { +func (i *ignition) GenerateNode(kluster *kubernikusv1.Kluster, pool *models.NodePool, nodeName, token string, secret *kubernikusv1.Secret, calicoNetworking bool, imageRegistry version.ImageRegistry, logger log.Logger) ([]byte, error) { ignition, err := i.getIgnitionTemplate(kluster) if err != nil { @@ -175,7 +175,7 @@ func (i *ignition) GenerateNode(kluster *kubernikusv1.Kluster, pool *models.Node KubeletClientsCA: secret.KubeletClientsCACertificate, ApiserverClientsSystemKubeProxy: secret.ApiserverClientsKubeProxyCertificate, ApiserverClientsSystemKubeProxyKey: secret.ApiserverClientsKubeProxyPrivateKey, - BootstrapToken: secret.BootstrapToken, + BootstrapToken: token, ClusterCIDR: kluster.ClusterCIDR(), ClusterDNSAddress: kluster.Spec.DNSAddress, ClusterDomain: kluster.Spec.DNSDomain, diff --git a/pkg/templates/ignition_test.go b/pkg/templates/ignition_test.go index ebeb0adb94..6874ca488a 100644 --- a/pkg/templates/ignition_test.go +++ b/pkg/templates/ignition_test.go @@ -96,7 +96,7 @@ func TestGenerateNode(t *testing.T) { for version := range imageRegistry.Versions { kluster.Spec.Version = version - data, err := Ignition.GenerateNode(kluster, nil, "test", &testKlusterSecret, false, imageRegistry, log.NewNopLogger()) + data, err := Ignition.GenerateNode(kluster, nil, "test", "abc123", &testKlusterSecret, false, imageRegistry, log.NewNopLogger()) if assert.NoError(t, err, "Failed to generate node for version %s", version) { //Ensure we rendered the expected template @@ -116,14 +116,14 @@ func TestNodeLabels(t *testing.T) { pool := &models.NodePool{Name: "some-name"} - data, err := Ignition.GenerateNode(kluster, pool, "test", &testKlusterSecret, false, imageRegistry, log.NewNopLogger()) + data, err := Ignition.GenerateNode(kluster, pool, "test", "abc123", &testKlusterSecret, false, imageRegistry, log.NewNopLogger()) if assert.NoError(t, err, "Failed to generate node") { //Ensure we rendered the expected template assert.Contains(t, string(data), fmt.Sprintf("--node-labels=ccloud.sap.com/nodepool=%s", pool.Name)) } gpuPool := &models.NodePool{Name: "some-name", Flavor: "zghuh"} - data, err = Ignition.GenerateNode(kluster, gpuPool, "test", &testKlusterSecret, false, imageRegistry, log.NewNopLogger()) + data, err = Ignition.GenerateNode(kluster, gpuPool, "test", "abc123", &testKlusterSecret, false, imageRegistry, log.NewNopLogger()) if assert.NoError(t, err, "Failed to generate node") { //Ensure we rendered the expected template assert.Contains(t, string(data), fmt.Sprintf("--node-labels=ccloud.sap.com/nodepool=%s,gpu=nvidia-tesla-v100", pool.Name)) diff --git a/pkg/util/bootstraptoken/util.go b/pkg/util/bootstraptoken/util.go index e0166bee97..7715b940ff 100644 --- a/pkg/util/bootstraptoken/util.go +++ b/pkg/util/bootstraptoken/util.go @@ -10,7 +10,7 @@ import ( bootstraptokenutil "k8s.io/cluster-bootstrap/token/util" ) -func GenerateBootstrapToken() (string, *corev1.Secret, error) { +func GenerateBootstrapToken(t time.Duration) (string, *corev1.Secret, error) { token, err := bootstraptokenutil.GenerateBootstrapToken() if err != nil { @@ -34,7 +34,7 @@ func GenerateBootstrapToken() (string, *corev1.Secret, error) { bootstraptokenapi.BootstrapTokenDescriptionKey: "Bootstrap token generated by Kubernikus", bootstraptokenapi.BootstrapTokenIDKey: tokenID, bootstraptokenapi.BootstrapTokenSecretKey: tokenSecret, - bootstraptokenapi.BootstrapTokenExpirationKey: metav1.Now().Add(24 * time.Hour).Format(time.RFC3339), + bootstraptokenapi.BootstrapTokenExpirationKey: metav1.Now().Add(t).Format(time.RFC3339), bootstraptokenapi.BootstrapTokenUsageAuthentication: "true", bootstraptokenapi.BootstrapTokenUsageSigningKey: "true", },