Use Key Vault secrets instead of cert-manager secret in AKS cluster

deployment/Dockerfile

@@ -33,7 +33,7 @@ RUN apt-get install -y kubectl
 RUN curl https://baltocdn.com/helm/signing.asc |  apt-key add -
 RUN echo "deb https://baltocdn.com/helm/stable/debian/ all main" | tee /etc/apt/sources.list.d/helm-stable-debian.list
 RUN apt-get update
-RUN apt-get install helm=3.5.0-1
+RUN apt-get install helm=3.14.0-1
 
 # Install kubelogin
 

deployment/bin/deploy

@@ -138,14 +138,14 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
 
     # Install cert-manager
 
-    echo "Installing cert-manager..."
+    # echo "Installing cert-manager..."
 
-    helm upgrade --install \
-        cert-manager \
-        --namespace pc \
-        --create-namespace \
-        --version v1.6.0 \
-        --set installCRDs=true jetstack/cert-manager
+    # helm upgrade --install \
+    #     cert-manager \
+    #     --namespace pc \
+    #     --create-namespace \
+    #     --version v1.6.0 \
+    #     --set installCRDs=true jetstack/cert-manager
 
     echo "==================="
     echo "==== STAC API ====="
@@ -191,7 +191,8 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
         --set controller.service.loadBalancerIP="${INGRESS_IP}" \
         --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-dns-label-name"="${DNS_LABEL}" \
         --wait \
-        --timeout 2m0s
+        --timeout 2m0s \
+        -f bin/nginx-values.yaml
 
     #########################
     # Deploy Azure Function #

deployment/bin/lib

@@ -43,6 +43,10 @@ function gather_tf_output() {
     export ENVIRONMENT=$(tf_output environment)
     export INGRESS_IP=$(tf_output ingress_ip)
     export DNS_LABEL=$(tf_output dns_label)
+    export AZURE_TENANT=$(tf_output tenant_id)
+    export KEYVAULT_NAME=$(tf_output secret_provider_keyvault_name)
+    export SECRET_PROVIDER_MANAGED_IDENTITY_ID=$(tf_output secret_provider_managed_identity_id)
+    export SECRET_PROVIDER_KEYVAULT_SECRET=$(tf_output secret_provider_keyvault_secret)
 
     export FUNCTION_APP_NAME=$(tf_output function_app_name)
 
@@ -53,7 +57,7 @@ function gather_tf_output() {
 
 function render_values() {
     echo "Rendering chart value files..."
-
+    
     bin/jinja ${TF_OUTPUT_FILE} ${TEMPLATE_PATH} ${DEPLOY_VALUES_FILE}
 }
 

deployment/bin/nginx-values.yaml

@@ -0,0 +1,16 @@
+controller:
+  podLabels:
+    azure.workload.identity/use: "true"
+  extraVolumes:
+    - name: secrets-store-inline
+      csi:
+        driver: secrets-store.csi.k8s.io
+        readOnly: true
+        volumeAttributes:
+          secretProviderClass: "keyvault"
+  extraVolumeMounts:
+    - name: secrets-store-inline
+      mountPath: "/mnt/secrets-store"
+      readOnly: true
+  extraArgs:
+    default-ssl-certificate: pc/planetarycomputer-test-certificate

deployment/helm/deploy-values.template.yaml

@@ -73,8 +73,8 @@ tiler:
 
   stac_api_url: http://planetary-computer-stac.pc.svc.cluster.local
   stac_api_href: stac/
-  # PCT sas needs to be accessed through api management
-  pc_sdk_sas_url: https://pct-sas-westeurope-staging-apim.azure-api.net/sas/token
+  # PCT sas needs to be accessed through Azure Front Door
+  pc_sdk_sas_url: https://planetarycomputer-test.microsoft.com/sas/token
   pc_sdk_subscription_key: "{{ tf.pc_sdk_subscription_key }}"
   vectortile_sa_base_url: https://pcvectortiles.blob.core.windows.net
 
@@ -111,21 +111,19 @@ pcingress:
     secretName: "pqe-tls-secret"
 
   certIssuer:
-    enabled: true
+    enabled: false
     privateKeySecretRef: "{{ tf.cluster_cert_issuer }}"
     server: "{{ tf.cluster_cert_server }}"
     issuerEmail: "[email protected]"
     secretName: "pqe-tls-secret"
 
   ingress:
     enabled: true
-    tlsHost: "{{ tf.dns_label }}.{{ tf.location }}.cloudapp.azure.com"
+    tlsHost: "planetarycomputer-test.microsoft.com"
     hosts:
-      - "{{ tf.dns_label }}.{{ tf.location }}.cloudapp.azure.com"
       - "planetarycomputer-test.microsoft.com"
     annotations:
       kubernetes.io/ingress.class: nginx
-      cert-manager.io/cluster-issuer: "{{ tf.cluster_cert_issuer }}-pcingress"
       nginx.ingress.kubernetes.io/rewrite-target: /$2
       nginx.ingress.kubernetes.io/use-regex: "true"
       nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"
@@ -146,3 +144,12 @@ postgres:
   user: "{{ tf.pg_user }}"
   password: "{{ tf.pg_password }}"
   dbName: "{{ tf.pg_database }}"
+
+secretProvider:
+  create: true
+  providerName: "keyvault"
+  userAssignedIdentityID: "{{ env.SECRET_PROVIDER_MANAGED_IDENTITY_ID }}"
+  tenantId: "{{ env.AZURE_TENANT }}"
+  keyvaultName: "{{ env.KEYVAULT_NAME }}"
+  keyvaultCertificateName: "{{ env.SECRET_PROVIDER_KEYVAULT_SECRET }}"
+  kubernetesCertificateSecretName: "{{ env.SECRET_PROVIDER_KEYVAULT_SECRET }}"

deployment/helm/pc-apis-ingress/templates/cluster_issuer.yaml

@@ -1,20 +0,0 @@
-{{- if .Values.pcingress.certIssuer.enabled -}}
-apiVersion: cert-manager.io/v1
-kind: ClusterIssuer
-metadata:
-  name: {{ .Values.pcingress.certIssuer.privateKeySecretRef }}-pcingress
-spec:
-  acme:
-    server: {{ .Values.pcingress.certIssuer.server }}
-    email: {{ .Values.pcingress.certIssuer.issuerEmail }}
-    privateKeySecretRef:
-      name: {{ .Values.pcingress.certIssuer.privateKeySecretRef }}
-    solvers:
-    - http01:
-        ingress:
-          class: nginx
-          podTemplate:
-            spec:
-              nodeSelector:
-                "kubernetes.io/os": linux
-{{- end }}

deployment/helm/pc-apis-ingress/templates/ingress.yaml

@@ -18,7 +18,7 @@ spec:
   tls:
   - hosts:
     - {{ .Values.pcingress.ingress.tlsHost }}
-    secretName: {{ .Values.pcingress.cert.secretName }}
+    secretName: {{ .Values.secretProvider.kubernetesCertificateSecretName }}
   rules:
   {{- range .Values.pcingress.ingress.hosts }}
     - host: {{ . }}

deployment/helm/pc-apis-ingress/templates/secret-provider.yaml

@@ -0,0 +1,28 @@
+{{- if .Values.secretProvider.create -}}
+apiVersion: secrets-store.csi.x-k8s.io/v1
+kind: SecretProviderClass
+metadata:
+  name: {{ .Values.secretProvider.providerName }}
+  namespace: {{ .Values.namespace }}
+spec:
+  provider: azure
+  secretObjects:
+  - secretName: {{ .Values.secretProvider.kubernetesCertificateSecretName }}
+    type: kubernetes.io/tls
+    data:
+    - objectName: {{ .Values.secretProvider.keyvaultCertificateName }}
+      key: tls.crt
+    - objectName: {{ .Values.secretProvider.keyvaultCertificateName }}
+      key: tls.key
+  parameters:
+    usePodIdentity: "false"
+    clientID: "{{ .Values.secretProvider.userAssignedIdentityID }}"
+    keyvaultName: "{{ .Values.secretProvider.keyvaultName }}"
+    tenantId: "{{ .Values.secretProvider.tenantId }}"
+    cloudName: ""
+    objects:  |
+      array:
+        - |
+          objectName: {{ .Values.secretProvider.keyvaultCertificateName }}
+          objectType: secret
+{{- end }}

deployment/helm/pc-apis-ingress/values.yaml

@@ -7,6 +7,15 @@ stac:
 tiler:
   enabled: true
 
+secretProvider:
+  create: true
+  providerName: "keyvault"
+  userAssignedIdentityID: ""
+  tenantId: ""
+  keyvaultName: ""
+  keyvaultCertificateName: ""
+  kubernetesCertificateSecretName: ""
+
 pcingress:
   services:
     stac:

deployment/terraform/resources/aks.tf

@@ -5,6 +5,19 @@ resource "azurerm_kubernetes_cluster" "pc" {
   dns_prefix          = "${local.prefix}-cluster"
   kubernetes_version  = var.k8s_version
 
+  key_vault_secrets_provider {
+    secret_rotation_enabled = true
+  }
+  oidc_issuer_enabled = true
+
+   # https://learn.microsoft.com/en-us/azure/aks/auto-upgrade-cluster#use-cluster-auto-upgrade
+  automatic_channel_upgrade = "rapid"
+
+  # https://learn.microsoft.com/en-us/azure/aks/auto-upgrade-node-os-image
+  node_os_channel_upgrade = "NodeImage"
+
+  image_cleaner_enabled = true
+
   default_node_pool {
     name                 = "agentpool"
     os_sku               = "AzureLinux"
@@ -35,3 +48,22 @@ resource "azurerm_role_assignment" "network" {
   role_definition_name = "Network Contributor"
   principal_id         = azurerm_kubernetes_cluster.pc.identity[0].principal_id
 }
+
+resource "azurerm_federated_identity_credential" "cluster" {
+  name                = "federated-id-${local.prefix}-${var.environment}"
+  resource_group_name = azurerm_kubernetes_cluster.pc.node_resource_group
+  audience            = ["api://AzureADTokenExchange"]
+  issuer              = azurerm_kubernetes_cluster.pc.oidc_issuer_url
+  subject             = "system:serviceaccount:pc:nginx-ingress-ingress-nginx"
+  parent_id           = "/subscriptions/a84a690d-585b-4c7c-80d9-851a48af5a50/resourceGroups/MC_pct-apis-westeurope-staging_rg_pct-apis-westeurope-staging-cluster_westeurope/providers/Microsoft.ManagedIdentity/userAssignedIdentities/azurekeyvaultsecretsprovider-pct-apis-westeurope-staging-cluster"
+  timeouts {}
+}
+
+# If you add a second azurerm provider and use a data block to reference this key vault
+# then the identity that deploys this has to have permissions over both subscriptions
+# This role assignment was created manually but the resource is left here as a reminder
+# resource "azurerm_role_assignment" "certificateAccess" {
+#   scope                = #REDACTED
+#   role_definition_name = #REDACTED
+#   principal_id         = azurerm_kubernetes_cluster.pc.key_vault_secrets_provider[0].secret_identity[0].object_id
+# }

deployment/terraform/resources/output.tf

@@ -14,6 +14,10 @@ output "resource_group" {
   value = azurerm_resource_group.pc.name
 }
 
+output "tenant_id" {
+  value = data.azurerm_client_config.current.tenant_id
+}
+
 # -- Postgres
 
 output "pg_host" {
@@ -57,6 +61,18 @@ output "dns_label" {
   value = azurerm_public_ip.pc.domain_name_label
 }
 
+output "secret_provider_keyvault_name" {
+  value = var.secret_provider_keyvault_name
+}
+
+output "secret_provider_managed_identity_id" {
+  value = azurerm_kubernetes_cluster.pc.key_vault_secrets_provider[0].secret_identity[0].client_id
+}
+
+output "secret_provider_keyvault_secret" {
+  value = var.secret_provider_keyvault_secret
+}
+
 ## STAC API
 
 output "stac_replica_count" {

deployment/terraform/resources/providers.tf

@@ -12,3 +12,5 @@ terraform {
     }
   }
 }
+
+data "azurerm_client_config" "current" {}

deployment/terraform/resources/variables.tf

@@ -84,6 +84,18 @@ variable "pc_sdk_subscription_key_secret_name" {
   default = "pct-tiler-sdk-subscription-key"
 }
 
+variable "secret_provider_keyvault_name" {
+  type = string
+  description = "The name of the KeyVault that holds the secrets"
+  default = "pc-deploy-secrets"
+}
+
+variable "secret_provider_keyvault_secret" {
+  type = string
+  description = "The name of the certificate in the KeyVault for TLS ingress"
+  default = "planetarycomputer-test-certificate"
+}
+
 # -- Functions --
 
 variable "output_storage_account_name" {