Use Key Vault secret provider for TLS certificates (#292)

deployment/bin/deploy

@@ -1,7 +1,6 @@
 #!/bin/bash
 
 set -e
-
 source bin/lib
 
 if [[ "${CI}" ]]; then
@@ -25,16 +24,12 @@ Options:
     --skip-tf-init: Skip running terrform init.
     --skip-functions: Don't run function publish.
     --skip-fetch-tf-vars: Skip fetching terraform variables.
+    --user-auth: Use mounted Azure user credentials
     -y: Skip confirmation.
 
 "
 }
 
-require_env "ARM_SUBSCRIPTION_ID"
-require_env "ARM_TENANT_ID"
-require_env "ARM_CLIENT_ID"
-require_env "ARM_USE_OIDC"
-
 
 ###################
 # Parse arguments #
@@ -79,6 +74,10 @@ while [[ "$#" -gt 0 ]]; do case $1 in
         SKIP_FETCH_TF_VARS=1
         shift
         ;;
+    --user-auth)
+        export USER_AUTH=1
+        shift
+        ;;
     --help)
         usage
         exit 0
@@ -100,6 +99,16 @@ if [[ -z ${TERRAFORM_DIR} ]]; then
     exit 1
 fi
 
+if [[ -z $USER_AUTH ]]; then
+    require_env "ARM_SUBSCRIPTION_ID"
+    require_env "ARM_TENANT_ID"
+    require_env "ARM_CLIENT_ID"
+    require_env "ARM_USE_OIDC"
+
+    require_env "AZURE_TENANT_ID"
+    require_env "AZURE_CLIENT_ID"
+fi
+
 setup_env
 
 # ---------------------------------------------------
@@ -204,19 +213,6 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
 
     setup_helm
 
-    # Install cert-manager
-
-    echo "==================="
-    echo "== cert-manager ==="
-    echo "==================="
-
-    helm upgrade --install \
-        cert-manager \
-        --namespace pc \
-        --create-namespace \
-        --version v1.6.0 \
-        --set installCRDs=true jetstack/cert-manager
-
     echo "=================="
     echo "===== Argo ======="
     echo "=================="
@@ -297,7 +293,8 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
         --set controller.service.loadBalancerIP="${INTERNAL_INGRESS_IP}" \
         --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-load-balancer-internal"=true \
         --wait \
-        --timeout 2m0s
+        --timeout 2m0s \
+        -f bin/nginx-values.yaml
 
 
     ######################

deployment/bin/lib

@@ -48,6 +48,10 @@ function gather_tf_output() {
     export INGRESS_IP=$(tf_output ingress_ip)
     export DNS_LABEL=$(tf_output dns_label)
     export INTERNAL_INGRESS_IP=$(tf_output internal_ingress_ip)
+    export AZURE_TENANT=$(tf_output tenant_id)
+    export KEYVAULT_NAME=$(tf_output secret_provider_keyvault_name)
+    export SECRET_PROVIDER_MANAGED_IDENTITY_ID=$(tf_output secret_provider_managed_identity_id)
+    export SECRET_PROVIDER_KEYVAULT_SECRET=$(tf_output secret_provider_keyvault_secret)
 
     if [ "${1}" ]; then
         popd
@@ -85,6 +89,8 @@ function cluster_login() {
     kubelogin convert-kubeconfig \
         -l azurecli \
         --kubeconfig=kubeconfig
+        # --client-id ${ARM_CLIENT_ID} \
+        # --client-secret ${ARM_CLIENT_SECRET} \
     export KUBECONFIG=kubeconfig
 }
 

deployment/bin/nginx-values.yaml

@@ -0,0 +1,16 @@
+controller:
+  podLabels:
+    azure.workload.identity/use: "true"
+  extraVolumes:
+    - name: secrets-store-inline
+      csi:
+        driver: secrets-store.csi.k8s.io
+        readOnly: true
+        volumeAttributes:
+          secretProviderClass: "keyvault"
+  extraVolumeMounts:
+    - name: secrets-store-inline
+      mountPath: "/mnt/secrets-store"
+      readOnly: true
+  extraArgs:
+    default-ssl-certificate: pc/planetarycomputer-test-certificate

deployment/helm/deploy-values.template.yaml

@@ -103,24 +103,24 @@ pcingress:
   cert:
     secretName: "pctasks-tls-secret"
 
-  certIssuer:
-    enabled: true
-    privateKeySecretRef: "{{ tf.cluster_cert_issuer }}"
-    server: "{{ tf.cluster_cert_server }}"
-    issuerEmail: "[email protected]"
-    secretName: "pctasks-tls-secret"
-
   ingress:
     enabled: true
-    tlsHost: "{{ tf.cloudapp_hostname }}"
+    tlsHost: "planetarycomputer-test.microsoft.com"
     hosts:
-      - "{{ tf.cloudapp_hostname }}"
       - "planetarycomputer-test.microsoft.com"
-      - "{{ tf.api_management_name }}.azure-api.net"
     annotations:
       kubernetes.io/ingress.class: nginx
-      cert-manager.io/cluster-issuer: "{{ tf.cluster_cert_issuer }}-pcingress"
       nginx.ingress.kubernetes.io/rewrite-target: /$2
       nginx.ingress.kubernetes.io/use-regex: "true"
       nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"
       nginx.ingress.kubernetes.io/proxy-buffers-number: "8"
+
+secretProvider:
+  create: true
+  namespace: "pc"
+  providerName: "keyvault"
+  userAssignedIdentityID: "{{ env.SECRET_PROVIDER_MANAGED_IDENTITY_ID }}"
+  tenantId: "{{ env.AZURE_TENANT }}"
+  keyvaultName: "{{ env.KEYVAULT_NAME }}"
+  keyvaultCertificateName: "{{ env.SECRET_PROVIDER_KEYVAULT_SECRET }}"
+  kubernetesCertificateSecretName: "{{ env.SECRET_PROVIDER_KEYVAULT_SECRET }}"

deployment/helm/pc-tasks-ingress/templates/NOTES.txt

@@ -2,4 +2,4 @@ Application information:
 {{ include "pcingress.selectorLabels" . }}
 Ingress host: {{ .Values.pcingress.ingress.host }}
 Service Fullname: {{ include "pcingress.fullname" . }}
-Cert enabled: {{ .Values.pcingress.certIssuer.enabled }}
+KeyVault secret provider created: {{ .Values.secretProvider.create }}

deployment/helm/pc-tasks-ingress/templates/ingress.yaml

@@ -18,7 +18,7 @@ spec:
   tls:
   - hosts:
     - {{ .Values.pcingress.ingress.tlsHost }}
-    secretName: {{ .Values.pcingress.cert.secretName }}
+    secretName: {{ .Values.secretProvider.kubernetesCertificateSecretName }}
   rules:
   {{- range .Values.pcingress.ingress.hosts }}
     - host: {{ . }}

deployment/helm/pc-tasks-ingress/templates/secret-provider.yaml

@@ -0,0 +1,28 @@
+{{- if .Values.secretProvider.create -}}
+apiVersion: secrets-store.csi.x-k8s.io/v1
+kind: SecretProviderClass
+metadata:
+  name: {{ .Values.secretProvider.providerName }}
+  namespace: {{ .Values.secretProvider.namespace }}
+spec:
+  provider: azure
+  secretObjects:
+  - secretName: {{ .Values.secretProvider.kubernetesCertificateSecretName }}
+    type: kubernetes.io/tls
+    data:
+    - objectName: {{ .Values.secretProvider.keyvaultCertificateName }}
+      key: tls.crt
+    - objectName: {{ .Values.secretProvider.keyvaultCertificateName }}
+      key: tls.key
+  parameters:
+    usePodIdentity: "false"
+    clientID: "{{ .Values.secretProvider.userAssignedIdentityID }}"
+    keyvaultName: "{{ .Values.secretProvider.keyvaultName }}"
+    tenantId: "{{ .Values.secretProvider.tenantId }}"
+    cloudName: ""
+    objects:  |
+      array:
+        - |
+          objectName: {{ .Values.secretProvider.keyvaultCertificateName }}
+          objectType: secret
+{{- end }}

deployment/helm/pc-tasks-ingress/values.yaml

@@ -21,17 +21,21 @@ pcingress:
   cert:
     secretName: ""
 
-  certIssuer:
-    enabled: false
-    privateKeySecretRef: "letsencrypt-staging"
-    server: "https://acme-staging-v02.api.letsencrypt.org/directory"
-    issuerEmail: ""
-
   ingress:
     enabled: false
     tlsHost: ""
     hosts: []
     annotations: {}
 
+secretProvider:
+  create: true
+  providerName: "keyvault"
+  namespace: ""
+  userAssignedIdentityID: ""
+  tenantId: ""
+  keyvaultName: ""
+  keyvaultCertificateName: ""
+  kubernetesCertificateSecretName: ""
+
 nameOverride: ""
 fullnameOverride: ""

deployment/terraform/batch_pool/providers.tf

@@ -10,7 +10,7 @@ terraform {
   required_providers {
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = "3.97.1"
+      version = "3.103.1"
     }
   }
 }

deployment/terraform/resources/aks.tf

@@ -5,6 +5,12 @@ resource "azurerm_kubernetes_cluster" "pctasks" {
   dns_prefix          = "${local.prefix}-cluster"
   kubernetes_version  = var.k8s_version
 
+  key_vault_secrets_provider {
+    secret_rotation_enabled = true
+  }
+  oidc_issuer_enabled = true
+  workload_identity_enabled = true
+
   oms_agent {
     log_analytics_workspace_id = azurerm_log_analytics_workspace.pctasks.id
   }
@@ -149,3 +155,29 @@ resource "azurerm_role_assignment" "network" {
   role_definition_name = "Network Contributor"
   principal_id         = azurerm_kubernetes_cluster.pctasks.identity[0].principal_id
 }
+
+# When you enable the key vault secrets provider block in an AKS cluster,
+# this identity is created in the node resource group. Altough it technically
+# is a property of the cluster resource under addProfiles.azureKeyvaultSecretsProvider.identity.resourceId
+# the terraform provider doesn't know about it so we need to manually tell terraform this thing exists
+data "azurerm_user_assigned_identity" "key_vault_secrets_provider_identity" {
+  resource_group_name = azurerm_kubernetes_cluster.pctasks.node_resource_group
+  name = "azurekeyvaultsecretsprovider-${azurerm_kubernetes_cluster.pctasks.name}"
+}
+
+resource "azurerm_federated_identity_credential" "cluster" {
+  name                = "federated-id-${local.prefix}-${var.environment}"
+  resource_group_name = azurerm_kubernetes_cluster.pctasks.node_resource_group
+  audience            = ["api://AzureADTokenExchange"]
+  issuer              = azurerm_kubernetes_cluster.pctasks.oidc_issuer_url
+  subject             = "system:serviceaccount:pc:nginx-ingress-ingress-nginx"
+  parent_id           = data.azurerm_user_assigned_identity.key_vault_secrets_provider_identity.id
+  timeouts {}
+}
+
+# Left here as an exercise for the reader in PIM elevation
+# resource "azurerm_role_assignment" "certificateAccess" {
+#   scope                = "/subscriptions/9da7523a-cb61-4c3e-b1d4-afa5fc6d2da9/resourceGroups/pc-manual-resources/providers/Microsoft.KeyVault/vaults/pc-deploy-secrets"
+#   role_definition_name = "Key Vault Secrets User"
+#   principal_id         = azurerm_kubernetes_cluster.pctasks.key_vault_secrets_provider[0].secret_identity[0].object_id
+# }

deployment/terraform/resources/output.tf

@@ -10,6 +10,24 @@ output "location" {
   value = local.location
 }
 
+output "tenant_id" {
+  value = data.azurerm_client_config.current.tenant_id
+}
+
+## Ingress
+
+output "secret_provider_keyvault_name" {
+  value = var.secret_provider_keyvault_name
+}
+
+output "secret_provider_managed_identity_id" {
+  value = azurerm_kubernetes_cluster.pctasks.key_vault_secrets_provider[0].secret_identity[0].client_id
+}
+
+output "secret_provider_keyvault_secret" {
+  value = var.secret_provider_keyvault_secret
+}
+
 ## AKS
 
 output "cluster_name" {

deployment/terraform/resources/providers.tf

@@ -10,7 +10,7 @@ terraform {
   required_providers {
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = "3.97.1"
+      version = "3.103.1"
     }
   }
 }

deployment/terraform/resources/variables.tf

@@ -6,6 +6,19 @@ variable "region" {
   type = string
 }
 
+# Ingress
+variable "secret_provider_keyvault_name" {
+  type = string
+  description = "The name of the KeyVault that holds the secrets"
+  default = "pc-deploy-secrets"
+}
+
+variable "secret_provider_keyvault_secret" {
+  type = string
+  description = "The name of the certificate in the KeyVault for TLS ingress"
+  default = "planetarycomputer-test-certificate"
+}
+
 # APIM
 
 variable "apim_sku_name" {

docker-compose.console.yml

@@ -7,6 +7,7 @@ services:
       dockerfile: Dockerfile.dev
     volumes:
       - .:/opt/src
+      - ~/.azure:/root/.azure
     environment:
       - PCTASKS_CLIENT__ENDPOINT=http://nginx/tasks
       - PCTASKS_CLIENT__API_KEY=hunter2
@@ -111,3 +112,4 @@ services:
 networks:
   default:
     name: pctasks-network
+    external: true

docker-compose.yml

@@ -2,7 +2,7 @@ version: "2.1"
 services:
   azurite:
     container_name: pctasks-azurite
-    image: mcr.microsoft.com/azure-storage/azurite:3.27.0
+    image: mcr.microsoft.com/azure-storage/azurite:3.30.0
     hostname: azurite
     command: "azurite --silent --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost
       0.0.0.0 -l /workspace"