Add support for leveraging pre-created secrets in flyte-binary helm c…

…hart (#3807) * Add support for leveraging pre-created secrets in flyte-binary helm chart Signed-off-by: Jeev B <[email protected]> * template clientSecretsExternalSecretRef Signed-off-by: Jeev B <[email protected]> * Store sensitive values in Secret object Signed-off-by: Jeev B <[email protected]> * Condition client secret hash on value being present Signed-off-by: Jeev B <[email protected]> * Quote db password Signed-off-by: Jeev B <[email protected]> * Update sandbox manifest Signed-off-by: Jeev B <[email protected]> * fix build Signed-off-by: Jeev B <[email protected]> * debug build Signed-off-by: Jeev B <[email protected]> * add back db password path Signed-off-by: Yee Hing Tong <[email protected]> * Drop link Signed-off-by: Jeev B <[email protected]> --------- Signed-off-by: Jeev B <[email protected]> Signed-off-by: Yee Hing Tong <[email protected]> Co-authored-by: Yee Hing Tong <[email protected]>
flyteorg · Jul 10, 2023 · 7102814 · 7102814
1 parent e6bd94a
commit 7102814
Show file tree

Hide file tree

Showing 11 changed files with 163 additions and 91 deletions.
diff --git a/charts/flyte-binary/README.md b/charts/flyte-binary/README.md
@@ -18,6 +18,7 @@ Chart for basic single Flyte executable deployment
 | configuration.agentService | object | `{}` |  |
 | configuration.annotations | object | `{}` |  |
 | configuration.auth.authorizedUris | list | `[]` |  |
+| configuration.auth.clientSecretsExternalSecretRef | string | `""` |  |
 | configuration.auth.enableAuthServer | bool | `true` |  |
 | configuration.auth.enabled | bool | `false` |  |
 | configuration.auth.flyteClient.audience | string | `""` |  |
@@ -40,8 +41,10 @@ Chart for basic single Flyte executable deployment
 | configuration.database.port | int | `5432` |  |
 | configuration.database.username | string | `"postgres"` |  |
 | configuration.externalConfigMap | string | `""` |  |
+| configuration.externalSecretRef | string | `""` |  |
 | configuration.inline | object | `{}` |  |
 | configuration.inlineConfigMap | string | `""` |  |
+| configuration.inlineSecretRef | string | `""` |  |
 | configuration.labels | object | `{}` |  |
 | configuration.logging.level | int | `1` |  |
 | configuration.logging.plugins.cloudwatch.enabled | bool | `false` |  |

diff --git a/charts/flyte-binary/templates/_helpers.tpl b/charts/flyte-binary/templates/_helpers.tpl
@@ -70,24 +70,24 @@ Create the name of the service account to use
 {{- end }}
 
 {{/*
-Get the Flyte configuration ConfigMap name.
+Flag to use external configuration.
 */}}
-{{- define "flyte-binary.configuration.configMapName" -}}
-{{- printf "%s-config" (include "flyte-binary.fullname" .) -}}
+{{- define "flyte-binary.configuration.externalConfiguration" -}}
+{{- or .Values.configuration.externalConfigMap .Values.configuration.externalSecretRef -}}
 {{- end -}}
 
 {{/*
-Get the Flyte configuration database password secret name.
+Get the Flyte configuration ConfigMap name.
 */}}
-{{- define "flyte-binary.configuration.database.passwordSecretName" -}}
-{{- printf "%s-db-pass" (include "flyte-binary.fullname" .) -}}
+{{- define "flyte-binary.configuration.configMapName" -}}
+{{- printf "%s-config" (include "flyte-binary.fullname" .) -}}
 {{- end -}}
 
 {{/*
-Get the Flyte configuration database password secret mount path.
+Get the Flyte configuration Secret name.
 */}}
-{{- define "flyte-binary.configuration.database.passwordSecretMountPath" -}}
-{{- default "/var/run/secrets/flyte/db-pass" .Values.configuration.database.passwordPath -}}
+{{- define "flyte-binary.configuration.configSecretName" -}}
+{{- printf "%s-config-secret" (include "flyte-binary.fullname" .) -}}
 {{- end -}}
 
 {{/*

diff --git a/charts/flyte-binary/templates/auth-client-secret.yaml b/charts/flyte-binary/templates/auth-client-secret.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.configuration.auth.enabled }}
+{{- if and .Values.configuration.auth.enabled (not .Values.configuration.auth.clientSecretsExternalSecretRef) }}
 apiVersion: v1
 kind: Secret
 metadata:

diff --git a/charts/flyte-binary/templates/config-secret.yaml b/charts/flyte-binary/templates/config-secret.yaml
@@ -0,0 +1,52 @@
+{{- if not (include "flyte-binary.configuration.externalConfiguration" .) }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "flyte-binary.configuration.configSecretName" . }}
+  namespace: {{ .Release.Namespace | quote }}
+  labels: {{- include "flyte-binary.labels" . | nindent 4 }}
+    {{- if .Values.commonLabels }}
+    {{- tpl ( .Values.commonLabels | toYaml ) . | nindent 4 }}
+    {{- end }}
+    {{- if .Values.configuration.labels }}
+    {{- tpl ( .Values.configuration.labels | toYaml ) . | nindent 4 }}
+    {{- end }}
+  annotations:
+    {{- if .Values.commonAnnotations }}
+    {{- tpl ( .Values.commonAnnotations | toYaml ) . | nindent 4 }}
+    {{- end }}
+    {{- if .Values.configuration.annotations }}
+    {{- tpl ( .Values.configuration.annotations | toYaml ) . | nindent 4 }}
+    {{- end }}
+type: Opaque
+stringData:
+  {{- if .Values.configuration.database.password }}
+  012-database-secrets.yaml: |
+    database:
+      postgres:
+        password: {{ .Values.configuration.database.password | quote }}
+  {{- end }}
+  {{- if eq "s3" .Values.configuration.storage.provider }}
+  {{- if eq "accesskey" .Values.configuration.storage.providerConfig.s3.authType }}
+  013-storage-secrets.yaml: |
+    storage:
+      stow:
+        config:
+          access_key_id: {{ required "Access key required for S3 storage provider" .Values.configuration.storage.providerConfig.s3.accessKey | quote }}
+          secret_key: {{ required "Secret key required for S3 storage provider" .Values.configuration.storage.providerConfig.s3.secretKey | quote }}
+  {{- end }}
+  {{- end }}
+  {{- if .Values.configuration.auth.enabled }}
+  {{- if .Values.configuration.auth.enableAuthServer }}
+  {{- if .Values.configuration.auth.internal.clientSecretHash }}
+  014-auth-secrets.yaml: |
+    auth:
+      appAuth:
+        selfAuthServer:
+          staticClients:
+            flytepropeller:
+              client_secret: {{ .Values.configuration.auth.internal.clientSecretHash | quote }}
+  {{- end }}
+  {{- end }}
+  {{- end }}
+{{- end }}
diff --git a/charts/flyte-binary/templates/configmap.yaml b/charts/flyte-binary/templates/configmap.yaml
@@ -1,4 +1,4 @@
-{{- if not .Values.configuration.externalConfigMap }}
+{{- if not (include "flyte-binary.configuration.externalConfiguration" .) }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -75,15 +75,13 @@ data:
     database:
       postgres:
         username: {{ .username }}
-        {{- if .password }}
-        passwordPath: {{ include "flyte-binary.configuration.database.passwordSecretMountPath" $ }}
-        {{- else }}
-        passwordPath: {{ .passwordPath }}
-        {{- end }}
         host: {{ tpl .host $ }}
         port: {{ .port }}
         dbname: {{ .dbname }}
         options: {{ .options | quote }}
+        {{- if .passwordPath }}
+        passwordPath: {{ .passwordPath }}
+        {{- end }}
     {{- end }}
   003-storage.yaml: |
     propeller:
@@ -106,8 +104,6 @@ data:
           auth_type: iam
           {{- else if eq "accesskey" .authType }}
           auth_type: accesskey
-          access_key_id: {{ required "Access key required for S3 storage provider" .accessKey }}
-          secret_key: {{ required "Secret key required for S3 storage provider" .secretKey }}
           {{- else }}
           {{- printf "Invalid value for S3 storage provider authentication type. Expected one of (iam, accesskey), but got: %s" .authType | fail }}
           {{- end }}
@@ -131,7 +127,6 @@ data:
         selfAuthServer:
           staticClients:
             flytepropeller:
-              client_secret: {{ required "Internal client secret hash required when built-in authentication server is enabled" .Values.configuration.auth.internal.clientSecretHash | quote }}
               grant_types:
               - refresh_token
               - client_credentials
@@ -197,7 +192,7 @@ data:
         useAuth: true
   {{- end }}
   {{- if .Values.configuration.inline }}
-  010-inline-config.yaml: |
+  100-inline-config.yaml: |
     {{- tpl ( .Values.configuration.inline | toYaml ) . | nindent 4 }}
   {{- end }}
 {{- end }}
diff --git a/charts/flyte-binary/templates/db-password-secret.yaml b/charts/flyte-binary/templates/db-password-secret.yaml
diff --git a/charts/flyte-binary/templates/deployment.yaml b/charts/flyte-binary/templates/deployment.yaml
@@ -33,19 +33,19 @@ spec:
         {{- tpl ( .Values.deployment.podLabels | toYaml ) . | nindent 8 }}
         {{- end }}
       annotations:
-        {{- if not .Values.configuration.existingConfigmap }}
+        {{- if not (include "flyte-binary.configuration.externalConfiguration" .) }}
         checksum/configuration: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+        checksum/configuration-secret: {{ include (print $.Template.BasePath "/config-secret.yaml") . | sha256sum }}
         {{- end }}
-        {{- if not .Values.clusterResourceTemplates.existingConfigmap }}
+        {{- if not .Values.clusterResourceTemplates.externalConfigMap }}
         checksum/cluster-resource-templates: {{ include (print $.Template.BasePath "/cluster-resource-templates.yaml") . | sha256sum }}
         {{- end }}
-        {{- if and .Values.configuration.database.password (not .Values.configuration.externalConfigMap) }}
-        checksum/db-password-secret: {{ include (print $.Template.BasePath "/db-password-secret.yaml") . | sha256sum }}
-        {{- end }}
         {{- if .Values.configuration.auth.enabled }}
         checksum/admin-auth-secret: {{ include (print $.Template.BasePath "/admin-auth-secret.yaml") . | sha256sum }}
+        {{- if not .Values.configuration.auth.clientSecretsExternalSecretRef }}
         checksum/auth-client-secret: {{ include (print $.Template.BasePath "/auth-client-secret.yaml") . | sha256sum }}
         {{- end }}
+        {{- end }}
         {{- if .Values.commonAnnotations }}
         {{- tpl ( .Values.commonAnnotations | toYaml ) . | nindent 4 }}
         {{- end }}
@@ -60,9 +60,9 @@ spec:
       securityContext: {{- omit .Values.deployment.podSecurityContext "enabled" | toYaml | nindent 12 }}
       {{- end }}
       serviceAccountName: {{ include "flyte-binary.serviceAccountName" . }}
-      {{- if or .Values.deployment.initContainers (not .Values.configuration.externalConfigMap) }}
+      {{- if or .Values.deployment.initContainers (not (include "flyte-binary.configuration.externalConfiguration" .)) }}
       initContainers:
-        {{- if not .Values.configuration.externalConfigMap }}
+        {{- if not (include "flyte-binary.configuration.externalConfiguration" .) }}
         - name: wait-for-db
           {{- with .Values.deployment.waitForDB.image }}
           image: {{ printf "%s:%s" .repository .tag | quote }}
@@ -217,11 +217,6 @@ spec:
               mountPath: /etc/flyte/cluster-resource-templates
             - name: config
               mountPath: /etc/flyte/config.d
-            {{- if and .Values.configuration.database.password (not .Values.configuration.externalConfigMap) }}
-            - name: db-pass
-              mountPath: {{ include "flyte-binary.configuration.database.passwordSecretMountPath" . }}
-              subPath: {{ include "flyte-binary.configuration.database.passwordSecretMountPath" . | base }}
-            {{- end }}
             - name: state
               mountPath: /var/run/flyte
             {{- if .Values.deployment.extraVolumeMounts }}
@@ -237,44 +232,56 @@ spec:
             sources:
             - secret:
                 name: {{ include "flyte-binary.configuration.auth.adminAuthSecretName" . }}
+            {{- if .Values.configuration.auth.clientSecretsExternalSecretRef }}
+            - secret:
+                name: {{ tpl .Values.configuration.auth.clientSecretsExternalSecretRef . }}
+            {{- else }}
             - secret:
                 name: {{ include "flyte-binary.configuration.auth.clientSecretName" . }}
+            {{- end }}
         {{- end }}
         - name: cluster-resource-templates
           {{- if .Values.clusterResourceTemplates.externalConfigMap }}
           configMap:
             name: {{ tpl .Values.clusterResourceTemplates.externalConfigMap . }}
-          {{- else if .Values.clusterResourceTemplates.inlineConfigMap }}
+          {{- else }}
           projected:
             sources:
             - configMap:
                 name: {{ include "flyte-binary.clusterResourceTemplates.configMapName" . }}
+            {{- if .Values.clusterResourceTemplates.inlineConfigMap }}
             - configMap:
                 name: {{ tpl .Values.clusterResourceTemplates.inlineConfigMap . }}
-          {{- else }}
-          configMap:
-            name: {{ include "flyte-binary.clusterResourceTemplates.configMapName" . }}
+            {{- end }}
           {{- end }}
         - name: config
-          {{- if .Values.configuration.externalConfigMap }}
-          configMap:
-            name: {{ tpl .Values.configuration.externalConfigMap . }}
-          {{- else if .Values.configuration.inlineConfigMap }}
+          {{- if (include "flyte-binary.configuration.externalConfiguration" .) }}
+          projected:
+            sources:
+            {{- if .Values.configuration.externalConfigMap }}
+            - configMap:
+                name: {{ tpl .Values.configuration.externalConfigMap . }}
+            {{- end }}
+            {{- if .Values.configuration.externalSecretRef }}
+            - secret:
+                name: {{ tpl .Values.configuration.externalSecretRef . }}
+            {{- end }}
+          {{- else }}
           projected:
             sources:
             - configMap:
                 name: {{ include "flyte-binary.configuration.configMapName" . }}
+            - secret:
+                name: {{ include "flyte-binary.configuration.configSecretName" . }}
+            {{- if .Values.configuration.inlineConfigMap }}
             - configMap:
                 name: {{ tpl .Values.configuration.inlineConfigMap . }}
-          {{- else }}
-          configMap:
-            name: {{ include "flyte-binary.configuration.configMapName" . }}
+            {{- end }}
+            {{- if .Values.configuration.inlineSecretRef }}
+            - secret:
+                name: {{ tpl .Values.configuration.inlineSecretRef . }}
+            {{- end }}
           {{- end }}
-        {{- if and .Values.configuration.database.password (not .Values.configuration.externalConfigMap) }}
-        - name: db-pass
-          secret:
-            secretName: {{ include "flyte-binary.configuration.database.passwordSecretName" . }}
-        {{- end }}
         - name: state
           emptyDir: {}
         {{- if .Values.deployment.extraVolumes }}

diff --git a/charts/flyte-binary/values.yaml b/charts/flyte-binary/values.yaml
@@ -139,6 +139,9 @@ configuration:
       audience: ""
     # authorizedUris Set of URIs that clients are allowed to visit the service on
     authorizedUris: []
+    # clientSecretExternalSecretRef Specify an existing, external Secret containing values for `client_secret` and `oidc_client_secret`.
+    # If set, a Secret will not be generated by this chart for client secrets.
+    clientSecretsExternalSecretRef: ""
   # co-pilot Configuration for Flyte CoPilot
   co-pilot:
     # image Configure image to use for CoPilot sidecar
@@ -154,13 +157,19 @@ configuration:
   #   supportedTaskTypes:
   #     - custom_task_type
   # externalConfigMap Specify an existing, external ConfigMap to use as configuration for Flyte
-  # If set, no ConfigMap will be generated by this chart
+  # If set, no Flyte configuration will be generated by this chart
   externalConfigMap: ""
+  # externalSecretRef Specify an existing, external Secret to use as configuration for Flyte
+  # If set, no Flyte configuration will be generated by this chart
+  externalSecretRef: ""
   # inline Specify additional configuration or overrides for Flyte, to be merged with the base configuration
   inline: {}
   # inlineConfigMap Specify an existing ConfigMap containing additional configuration
   # or overrides for Flyte, to be merged with the base configuration
   inlineConfigMap: ""
+  # inlineSecretRef Specify an existing Secret containing additional configuration
+  # or overrides for Flyte, to be merged with the base configuration
+  inlineSecretRef: ""
   # labels Add labels to created ConfigMap
   labels: {}
   # annotations Add annotations to created ConfigMap

diff --git a/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml b/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml
@@ -13,6 +13,22 @@ metadata:
     app.kubernetes.io/managed-by: Helm
   annotations:
 ---
+# Source: flyte-binary/templates/config-secret.yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: flyte-flyte-binary-config-secret
+  namespace: "flyte"
+  labels:
+    helm.sh/chart: flyte-binary-v0.1.10
+    app.kubernetes.io/name: flyte-binary
+    app.kubernetes.io/instance: flyte
+    app.kubernetes.io/version: "1.16.0"
+    app.kubernetes.io/managed-by: Helm
+  annotations:
+type: Opaque
+stringData:
+---
 # Source: flyte-binary/templates/cluster-resource-templates.yaml
 apiVersion: v1
 kind: ConfigMap
@@ -110,7 +126,6 @@ data:
     database:
       postgres:
         username: postgres
-        passwordPath: 
         host: 127.0.0.1
         port: 5432
         dbname: flyte
@@ -342,7 +357,8 @@ spec:
         app.kubernetes.io/instance: flyte
         app.kubernetes.io/component: flyte-binary
       annotations:
-        checksum/configuration: b2fb0938b3a627d086ba056e8769d9c683f64aaed96765330ffa1478b64f6a1f
+        checksum/configuration: ee7f2b8f30de7a3b54b5d3939ae877f8df9564627b84aaeed56d11d9f2fb2434
+        checksum/configuration-secret: d5d93f4e67780b21593dc3799f0f6682aab0765e708e4020939975d14d44f929
         checksum/cluster-resource-templates: 7dfa59f3d447e9c099b8f8ffad3af466fecbc9cf9f8c97295d9634254a55d4ae
     spec:
       serviceAccountName: flyte-flyte-binary
@@ -404,10 +420,16 @@ spec:
               mountPath: /var/run/flyte
       volumes:
         - name: cluster-resource-templates
-          configMap:
-            name: flyte-flyte-binary-cluster-resource-templates
+          projected:
+            sources:
+            - configMap:
+                name: flyte-flyte-binary-cluster-resource-templates
         - name: config
-          configMap:
-            name: flyte-flyte-binary-config
+          projected:
+            sources:
+            - configMap:
+                name: flyte-flyte-binary-config
+            - secret:
+                name: flyte-flyte-binary-config-secret
         - name: state
           emptyDir: {}