feat: Make kube-apiserver endpoint configurable

siderolabs · Oct 1, 2024 · 87a391a · 87a391a
1 parent 41fb283
commit 87a391a
Show file tree

Hide file tree

Showing 7 changed files with 62 additions and 19 deletions.
diff --git a/Makefile b/Makefile
@@ -122,11 +122,11 @@ docs:
 		charts/talos-cloud-controller-manager > docs/deploy/cloud-controller-manager-edge.yml
 	helm template -n kube-system talos-cloud-controller-manager \
 		--set-string image.tag=$(TAG) \
-		--set useDaemonSet=true \
+		--set daemonset.enabled=true \
 		charts/talos-cloud-controller-manager > docs/deploy/cloud-controller-manager-daemonset.yml
 	helm template -n kube-system talos-cloud-controller-manager \
 		-f charts/talos-cloud-controller-manager/values.edge.yaml \
-		--set useDaemonSet=true \
+		--set daemonset.enabled=true \
 		charts/talos-cloud-controller-manager > docs/deploy/cloud-controller-manager-daemonset-edge.yml
 	helm-docs charts/talos-cloud-controller-manager
 

diff --git a/charts/talos-cloud-controller-manager/README.md b/charts/talos-cloud-controller-manager/README.md
@@ -82,6 +82,9 @@ helm upgrade -i --namespace=kube-system -f talos-ccm.yaml \
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | affinity | object | `{}` | Affinity for data pods assignment. ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity |
+| daemonSet | object | `{"enabled":true,"k8s":{"serviceHost":"","servicePort":6443}}` | Deploy CCM  in Daemonset mode. CCM will use hostNetwork and connect to the Kubernetes API server on the current node by default. Optionally you can specify the Kubernetes API server host and port. You can run it without CNI plugin. |
+| daemonSet.k8s.serviceHost | string | `""` | Kubernetes API server host. Default is the current node IP. |
+| daemonSet.k8s.servicePort | int | `6443` | Kubernetes API server port. Default is 6443. |
 | enabledControllers | list | `["cloud-node","node-csr-approval"]` | List of controllers should be enabled. Use '*' to enable all controllers. Support only `cloud-node, cloud-node-lifecycle, node-csr-approval, node-ipam-controller` controllers. |
 | extraArgs | list | `[]` | Any extra arguments for talos-cloud-controller-manager |
 | fullnameOverride | string | `""` | String to fully override deployment name. |
@@ -108,4 +111,3 @@ helm upgrade -i --namespace=kube-system -f talos-ccm.yaml \
 | tolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/control-plane","operator":"Exists"},{"effect":"NoSchedule","key":"node.cloudprovider.kubernetes.io/uninitialized","operator":"Exists"}]` | Tolerations for data pods assignment. ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ |
 | transformations | list | `[]` | List of node transformations. Available matchExpressions key values: https://github.com/siderolabs/talos/blob/main/pkg/machinery/resources/runtime/platform_metadata.go#L28 |
 | updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":1},"type":"RollingUpdate"}` | Deployment update stategy type. ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment |
-| useDaemonSet | bool | `false` | Deploy CCM  in Daemonset mode. CCM will use hostNetwork and current node to access kubernetes/talos API You can run it without CNI plugin. |
diff --git a/charts/talos-cloud-controller-manager/templates/deployment.yaml b/charts/talos-cloud-controller-manager/templates/deployment.yaml
@@ -1,5 +1,5 @@
 apiVersion: apps/v1
-{{- if .Values.useDaemonSet }}
+{{- if .Values.daemonSet.enabled }}
 kind: DaemonSet
 {{- else }}
 kind: Deployment
@@ -10,7 +10,7 @@ metadata:
     {{- include "talos-cloud-controller-manager.labels" . | nindent 4 }}
   namespace: {{ .Release.Namespace }}
 spec:
-  {{- if not .Values.useDaemonSet }}
+  {{- if not .Values.daemonSet.enabled }}
   replicas: {{ .Values.replicaCount }}
   strategy:
     type: {{ .Values.updateStrategy.type }}
@@ -37,7 +37,7 @@ spec:
       serviceAccountName: {{ include "talos-cloud-controller-manager.serviceAccountName" . }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
-      {{- if .Values.useDaemonSet }}
+      {{- if .Values.daemonSet.enabled }}
       dnsPolicy: ClusterFirstWithHostNet
       hostNetwork: true
       {{- end }}
@@ -61,18 +61,22 @@ spec:
           {{- with .Values.extraArgs }}
             {{- toYaml . | nindent 12 }}
           {{- end }}
-          {{- if .Values.useDaemonSet }}
+          {{- if .Values.daemonSet.enabled }}
           env:
             - name: TALOS_ENDPOINTS
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
             - name: KUBERNETES_SERVICE_HOST
+              {{- if .Values.daemonSet.k8s.serviceHost }}
+              value: {{ .Values.daemonSet.k8s.serviceHost }}
+              {{- else }}
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+              {{- end }}
             - name: KUBERNETES_SERVICE_PORT
-              value: "6443"
+              value: {{ .Values.daemonSet.k8s.servicePort }}
           {{- end }}
           ports:
             - name: metrics
@@ -107,7 +111,7 @@ spec:
       {{- with .Values.tolerations }}
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- if .Values.useDaemonSet }}
+      {{- if .Values.daemonSet.enabled }}
         - effect: NoSchedule
           key: node.kubernetes.io/not-ready
           operator: Exists

diff --git a/charts/talos-cloud-controller-manager/values-example.yaml b/charts/talos-cloud-controller-manager/values-example.yaml
@@ -1,4 +1,3 @@
-
 # Use latest Talos image
 image:
   pullPolicy: Always
@@ -53,7 +52,8 @@ transformations:
       node-role.kubernetes.io/db: ""
 
 # Deploy the Talos Cloud Controller Manager as a DaemonSet
-useDaemonSet: true
+daemonSet:
+  enabled: true
 
 # Tolerate all taints
 tolerations:

diff --git a/charts/talos-cloud-controller-manager/values.yaml b/charts/talos-cloud-controller-manager/values.yaml
@@ -125,9 +125,16 @@ resources:
     memory: 64Mi
 
 # -- Deploy CCM  in Daemonset mode.
-# CCM will use hostNetwork and current node to access kubernetes/talos API
+# CCM will use hostNetwork and connect to the Kubernetes API server on the current node by default.
+# Optionally you can specify the Kubernetes API server host and port.
 # You can run it without CNI plugin.
-useDaemonSet: false
+daemonSet:
+  enabled: true
+  k8s:
+    # -- Kubernetes API server host. Default is the current node IP.
+    serviceHost: ""
+    # -- Kubernetes API server port. Default is 6443.
+    servicePort: 6443
 
 # -- Deployment update stategy type.
 # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment

diff --git a/docs/deploy/cloud-controller-manager-edge.yml b/docs/deploy/cloud-controller-manager-edge.yml
@@ -178,7 +178,7 @@ spec:
 ---
 # Source: talos-cloud-controller-manager/templates/deployment.yaml
 apiVersion: apps/v1
-kind: Deployment
+kind: DaemonSet
 metadata:
   name: talos-cloud-controller-manager
   labels:
@@ -189,8 +189,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
   namespace: kube-system
 spec:
-  replicas: 1
-  strategy:
+  updateStrategy:
     type: RollingUpdate
   selector:
     matchLabels:
@@ -209,6 +208,8 @@ spec:
         runAsGroup: 10258
         runAsNonRoot: true
         runAsUser: 10258
+      dnsPolicy: ClusterFirstWithHostNet
+      hostNetwork: true
       priorityClassName: system-cluster-critical
       containers:
         - name: talos-cloud-controller-manager
@@ -231,6 +232,17 @@ spec:
             - --use-service-account-credentials
             - --secure-port=50258
             - --authorization-always-allow-paths=/healthz,/livez,/readyz,/metrics
+          env:
+            - name: TALOS_ENDPOINTS
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: KUBERNETES_SERVICE_HOST
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: KUBERNETES_SERVICE_PORT
+              value: 6443
           ports:
             - name: metrics
               containerPort: 50258
@@ -263,6 +275,9 @@ spec:
         - effect: NoSchedule
           key: node.cloudprovider.kubernetes.io/uninitialized
           operator: Exists
+        - effect: NoSchedule
+          key: node.kubernetes.io/not-ready
+          operator: Exists
       volumes:
         - name: cloud-config
           configMap:

diff --git a/docs/deploy/cloud-controller-manager.yml b/docs/deploy/cloud-controller-manager.yml
@@ -178,7 +178,7 @@ spec:
 ---
 # Source: talos-cloud-controller-manager/templates/deployment.yaml
 apiVersion: apps/v1
-kind: Deployment
+kind: DaemonSet
 metadata:
   name: talos-cloud-controller-manager
   labels:
@@ -189,8 +189,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
   namespace: kube-system
 spec:
-  replicas: 1
-  strategy:
+  updateStrategy:
     type: RollingUpdate
   selector:
     matchLabels:
@@ -209,6 +208,8 @@ spec:
         runAsGroup: 10258
         runAsNonRoot: true
         runAsUser: 10258
+      dnsPolicy: ClusterFirstWithHostNet
+      hostNetwork: true
       priorityClassName: system-cluster-critical
       containers:
         - name: talos-cloud-controller-manager
@@ -231,6 +232,17 @@ spec:
             - --use-service-account-credentials
             - --secure-port=50258
             - --authorization-always-allow-paths=/healthz,/livez,/readyz,/metrics
+          env:
+            - name: TALOS_ENDPOINTS
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: KUBERNETES_SERVICE_HOST
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: KUBERNETES_SERVICE_PORT
+              value: 6443
           ports:
             - name: metrics
               containerPort: 50258
@@ -263,6 +275,9 @@ spec:
         - effect: NoSchedule
           key: node.cloudprovider.kubernetes.io/uninitialized
           operator: Exists
+        - effect: NoSchedule
+          key: node.kubernetes.io/not-ready
+          operator: Exists
       volumes:
         - name: cloud-config
           configMap: