From e7677068fec371fc495b953f284f3a344e6ec40a Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Wed, 15 May 2024 17:10:40 -0400 Subject: [PATCH] Use Key Vault secret provider for TLS certificates (#292) --- deployment/bin/deploy | 37 +++++++++---------- deployment/bin/lib | 6 +++ deployment/bin/nginx-values.yaml | 16 ++++++++ deployment/helm/deploy-values.template.yaml | 22 +++++------ .../helm/pc-tasks-ingress/templates/NOTES.txt | 2 +- .../templates/cluster_issuer.yaml | 20 ---------- .../pc-tasks-ingress/templates/ingress.yaml | 2 +- .../templates/secret-provider.yaml | 28 ++++++++++++++ deployment/helm/pc-tasks-ingress/values.yaml | 16 +++++--- deployment/terraform/batch_pool/providers.tf | 2 +- deployment/terraform/resources/aks.tf | 32 ++++++++++++++++ deployment/terraform/resources/output.tf | 18 +++++++++ deployment/terraform/resources/providers.tf | 2 +- deployment/terraform/resources/variables.tf | 13 +++++++ docker-compose.console.yml | 2 + docker-compose.yml | 2 +- 16 files changed, 158 insertions(+), 62 deletions(-) create mode 100644 deployment/bin/nginx-values.yaml delete mode 100644 deployment/helm/pc-tasks-ingress/templates/cluster_issuer.yaml create mode 100644 deployment/helm/pc-tasks-ingress/templates/secret-provider.yaml diff --git a/deployment/bin/deploy b/deployment/bin/deploy index e1d4d905..fad0abe5 100755 --- a/deployment/bin/deploy +++ b/deployment/bin/deploy @@ -1,7 +1,6 @@ #!/bin/bash set -e - source bin/lib if [[ "${CI}" ]]; then @@ -25,16 +24,12 @@ Options: --skip-tf-init: Skip running terrform init. --skip-functions: Don't run function publish. --skip-fetch-tf-vars: Skip fetching terraform variables. + --user-auth: Use mounted Azure user credentials -y: Skip confirmation. " } -require_env "ARM_SUBSCRIPTION_ID" -require_env "ARM_TENANT_ID" -require_env "ARM_CLIENT_ID" -require_env "ARM_USE_OIDC" - ################### # Parse arguments # @@ -79,6 +74,10 @@ while [[ "$#" -gt 0 ]]; do case $1 in SKIP_FETCH_TF_VARS=1 shift ;; + --user-auth) + export USER_AUTH=1 + shift + ;; --help) usage exit 0 @@ -100,6 +99,16 @@ if [[ -z ${TERRAFORM_DIR} ]]; then exit 1 fi +if [[ -z $USER_AUTH ]]; then + require_env "ARM_SUBSCRIPTION_ID" + require_env "ARM_TENANT_ID" + require_env "ARM_CLIENT_ID" + require_env "ARM_USE_OIDC" + + require_env "AZURE_TENANT_ID" + require_env "AZURE_CLIENT_ID" +fi + setup_env # --------------------------------------------------- @@ -204,19 +213,6 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then setup_helm - # Install cert-manager - - echo "===================" - echo "== cert-manager ===" - echo "===================" - - helm upgrade --install \ - cert-manager \ - --namespace pc \ - --create-namespace \ - --version v1.6.0 \ - --set installCRDs=true jetstack/cert-manager - echo "==================" echo "===== Argo =======" echo "==================" @@ -297,7 +293,8 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then --set controller.service.loadBalancerIP="${INTERNAL_INGRESS_IP}" \ --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-load-balancer-internal"=true \ --wait \ - --timeout 2m0s + --timeout 2m0s \ + -f bin/nginx-values.yaml ###################### diff --git a/deployment/bin/lib b/deployment/bin/lib index 9a667a64..4a79495a 100755 --- a/deployment/bin/lib +++ b/deployment/bin/lib @@ -48,6 +48,10 @@ function gather_tf_output() { export INGRESS_IP=$(tf_output ingress_ip) export DNS_LABEL=$(tf_output dns_label) export INTERNAL_INGRESS_IP=$(tf_output internal_ingress_ip) + export AZURE_TENANT=$(tf_output tenant_id) + export KEYVAULT_NAME=$(tf_output secret_provider_keyvault_name) + export SECRET_PROVIDER_MANAGED_IDENTITY_ID=$(tf_output secret_provider_managed_identity_id) + export SECRET_PROVIDER_KEYVAULT_SECRET=$(tf_output secret_provider_keyvault_secret) if [ "${1}" ]; then popd @@ -85,6 +89,8 @@ function cluster_login() { kubelogin convert-kubeconfig \ -l azurecli \ --kubeconfig=kubeconfig + # --client-id ${ARM_CLIENT_ID} \ + # --client-secret ${ARM_CLIENT_SECRET} \ export KUBECONFIG=kubeconfig } diff --git a/deployment/bin/nginx-values.yaml b/deployment/bin/nginx-values.yaml new file mode 100644 index 00000000..ab3dbc5a --- /dev/null +++ b/deployment/bin/nginx-values.yaml @@ -0,0 +1,16 @@ +controller: + podLabels: + azure.workload.identity/use: "true" + extraVolumes: + - name: secrets-store-inline + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "keyvault" + extraVolumeMounts: + - name: secrets-store-inline + mountPath: "/mnt/secrets-store" + readOnly: true + extraArgs: + default-ssl-certificate: pc/planetarycomputer-test-certificate \ No newline at end of file diff --git a/deployment/helm/deploy-values.template.yaml b/deployment/helm/deploy-values.template.yaml index 9f25ae46..5ce6ce95 100644 --- a/deployment/helm/deploy-values.template.yaml +++ b/deployment/helm/deploy-values.template.yaml @@ -103,24 +103,24 @@ pcingress: cert: secretName: "pctasks-tls-secret" - certIssuer: - enabled: true - privateKeySecretRef: "{{ tf.cluster_cert_issuer }}" - server: "{{ tf.cluster_cert_server }}" - issuerEmail: "planetarycomputer@microsoft.com" - secretName: "pctasks-tls-secret" - ingress: enabled: true - tlsHost: "{{ tf.cloudapp_hostname }}" + tlsHost: "planetarycomputer-test.microsoft.com" hosts: - - "{{ tf.cloudapp_hostname }}" - "planetarycomputer-test.microsoft.com" - - "{{ tf.api_management_name }}.azure-api.net" annotations: kubernetes.io/ingress.class: nginx - cert-manager.io/cluster-issuer: "{{ tf.cluster_cert_issuer }}-pcingress" nginx.ingress.kubernetes.io/rewrite-target: /$2 nginx.ingress.kubernetes.io/use-regex: "true" nginx.ingress.kubernetes.io/proxy-buffer-size: "16k" nginx.ingress.kubernetes.io/proxy-buffers-number: "8" + +secretProvider: + create: true + namespace: "pc" + providerName: "keyvault" + userAssignedIdentityID: "{{ env.SECRET_PROVIDER_MANAGED_IDENTITY_ID }}" + tenantId: "{{ env.AZURE_TENANT }}" + keyvaultName: "{{ env.KEYVAULT_NAME }}" + keyvaultCertificateName: "{{ env.SECRET_PROVIDER_KEYVAULT_SECRET }}" + kubernetesCertificateSecretName: "{{ env.SECRET_PROVIDER_KEYVAULT_SECRET }}" diff --git a/deployment/helm/pc-tasks-ingress/templates/NOTES.txt b/deployment/helm/pc-tasks-ingress/templates/NOTES.txt index 06bf7003..d97690e9 100644 --- a/deployment/helm/pc-tasks-ingress/templates/NOTES.txt +++ b/deployment/helm/pc-tasks-ingress/templates/NOTES.txt @@ -2,4 +2,4 @@ Application information: {{ include "pcingress.selectorLabels" . }} Ingress host: {{ .Values.pcingress.ingress.host }} Service Fullname: {{ include "pcingress.fullname" . }} -Cert enabled: {{ .Values.pcingress.certIssuer.enabled }} \ No newline at end of file +KeyVault secret provider created: {{ .Values.secretProvider.create }} \ No newline at end of file diff --git a/deployment/helm/pc-tasks-ingress/templates/cluster_issuer.yaml b/deployment/helm/pc-tasks-ingress/templates/cluster_issuer.yaml deleted file mode 100644 index 36ef27eb..00000000 --- a/deployment/helm/pc-tasks-ingress/templates/cluster_issuer.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{- if .Values.pcingress.certIssuer.enabled -}} -apiVersion: cert-manager.io/v1 -kind: ClusterIssuer -metadata: - name: {{ .Values.pcingress.certIssuer.privateKeySecretRef }}-pcingress -spec: - acme: - server: {{ .Values.pcingress.certIssuer.server }} - email: {{ .Values.pcingress.certIssuer.issuerEmail }} - privateKeySecretRef: - name: {{ .Values.pcingress.certIssuer.privateKeySecretRef }} - solvers: - - http01: - ingress: - class: nginx - podTemplate: - spec: - nodeSelector: - "kubernetes.io/os": linux -{{- end }} \ No newline at end of file diff --git a/deployment/helm/pc-tasks-ingress/templates/ingress.yaml b/deployment/helm/pc-tasks-ingress/templates/ingress.yaml index 1eb71f05..d6184e55 100644 --- a/deployment/helm/pc-tasks-ingress/templates/ingress.yaml +++ b/deployment/helm/pc-tasks-ingress/templates/ingress.yaml @@ -18,7 +18,7 @@ spec: tls: - hosts: - {{ .Values.pcingress.ingress.tlsHost }} - secretName: {{ .Values.pcingress.cert.secretName }} + secretName: {{ .Values.secretProvider.kubernetesCertificateSecretName }} rules: {{- range .Values.pcingress.ingress.hosts }} - host: {{ . }} diff --git a/deployment/helm/pc-tasks-ingress/templates/secret-provider.yaml b/deployment/helm/pc-tasks-ingress/templates/secret-provider.yaml new file mode 100644 index 00000000..e306d80f --- /dev/null +++ b/deployment/helm/pc-tasks-ingress/templates/secret-provider.yaml @@ -0,0 +1,28 @@ +{{- if .Values.secretProvider.create -}} +apiVersion: secrets-store.csi.x-k8s.io/v1 +kind: SecretProviderClass +metadata: + name: {{ .Values.secretProvider.providerName }} + namespace: {{ .Values.secretProvider.namespace }} +spec: + provider: azure + secretObjects: + - secretName: {{ .Values.secretProvider.kubernetesCertificateSecretName }} + type: kubernetes.io/tls + data: + - objectName: {{ .Values.secretProvider.keyvaultCertificateName }} + key: tls.crt + - objectName: {{ .Values.secretProvider.keyvaultCertificateName }} + key: tls.key + parameters: + usePodIdentity: "false" + clientID: "{{ .Values.secretProvider.userAssignedIdentityID }}" + keyvaultName: "{{ .Values.secretProvider.keyvaultName }}" + tenantId: "{{ .Values.secretProvider.tenantId }}" + cloudName: "" + objects: | + array: + - | + objectName: {{ .Values.secretProvider.keyvaultCertificateName }} + objectType: secret +{{- end }} \ No newline at end of file diff --git a/deployment/helm/pc-tasks-ingress/values.yaml b/deployment/helm/pc-tasks-ingress/values.yaml index af268956..4b9e5763 100644 --- a/deployment/helm/pc-tasks-ingress/values.yaml +++ b/deployment/helm/pc-tasks-ingress/values.yaml @@ -21,17 +21,21 @@ pcingress: cert: secretName: "" - certIssuer: - enabled: false - privateKeySecretRef: "letsencrypt-staging" - server: "https://acme-staging-v02.api.letsencrypt.org/directory" - issuerEmail: "" - ingress: enabled: false tlsHost: "" hosts: [] annotations: {} +secretProvider: + create: true + providerName: "keyvault" + namespace: "" + userAssignedIdentityID: "" + tenantId: "" + keyvaultName: "" + keyvaultCertificateName: "" + kubernetesCertificateSecretName: "" + nameOverride: "" fullnameOverride: "" diff --git a/deployment/terraform/batch_pool/providers.tf b/deployment/terraform/batch_pool/providers.tf index 7e2bead8..bec4ae60 100644 --- a/deployment/terraform/batch_pool/providers.tf +++ b/deployment/terraform/batch_pool/providers.tf @@ -10,7 +10,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "3.97.1" + version = "3.103.1" } } } diff --git a/deployment/terraform/resources/aks.tf b/deployment/terraform/resources/aks.tf index 55b7e5ed..6c6bd374 100644 --- a/deployment/terraform/resources/aks.tf +++ b/deployment/terraform/resources/aks.tf @@ -5,6 +5,12 @@ resource "azurerm_kubernetes_cluster" "pctasks" { dns_prefix = "${local.prefix}-cluster" kubernetes_version = var.k8s_version + key_vault_secrets_provider { + secret_rotation_enabled = true + } + oidc_issuer_enabled = true + workload_identity_enabled = true + oms_agent { log_analytics_workspace_id = azurerm_log_analytics_workspace.pctasks.id } @@ -149,3 +155,29 @@ resource "azurerm_role_assignment" "network" { role_definition_name = "Network Contributor" principal_id = azurerm_kubernetes_cluster.pctasks.identity[0].principal_id } + +# When you enable the key vault secrets provider block in an AKS cluster, +# this identity is created in the node resource group. Altough it technically +# is a property of the cluster resource under addProfiles.azureKeyvaultSecretsProvider.identity.resourceId +# the terraform provider doesn't know about it so we need to manually tell terraform this thing exists +data "azurerm_user_assigned_identity" "key_vault_secrets_provider_identity" { + resource_group_name = azurerm_kubernetes_cluster.pctasks.node_resource_group + name = "azurekeyvaultsecretsprovider-${azurerm_kubernetes_cluster.pctasks.name}" +} + +resource "azurerm_federated_identity_credential" "cluster" { + name = "federated-id-${local.prefix}-${var.environment}" + resource_group_name = azurerm_kubernetes_cluster.pctasks.node_resource_group + audience = ["api://AzureADTokenExchange"] + issuer = azurerm_kubernetes_cluster.pctasks.oidc_issuer_url + subject = "system:serviceaccount:pc:nginx-ingress-ingress-nginx" + parent_id = data.azurerm_user_assigned_identity.key_vault_secrets_provider_identity.id + timeouts {} +} + +# Left here as an exercise for the reader in PIM elevation +# resource "azurerm_role_assignment" "certificateAccess" { +# scope = "/subscriptions/9da7523a-cb61-4c3e-b1d4-afa5fc6d2da9/resourceGroups/pc-manual-resources/providers/Microsoft.KeyVault/vaults/pc-deploy-secrets" +# role_definition_name = "Key Vault Secrets User" +# principal_id = azurerm_kubernetes_cluster.pctasks.key_vault_secrets_provider[0].secret_identity[0].object_id +# } \ No newline at end of file diff --git a/deployment/terraform/resources/output.tf b/deployment/terraform/resources/output.tf index 2761d673..076e283b 100644 --- a/deployment/terraform/resources/output.tf +++ b/deployment/terraform/resources/output.tf @@ -10,6 +10,24 @@ output "location" { value = local.location } +output "tenant_id" { + value = data.azurerm_client_config.current.tenant_id +} + +## Ingress + +output "secret_provider_keyvault_name" { + value = var.secret_provider_keyvault_name +} + +output "secret_provider_managed_identity_id" { + value = azurerm_kubernetes_cluster.pctasks.key_vault_secrets_provider[0].secret_identity[0].client_id +} + +output "secret_provider_keyvault_secret" { + value = var.secret_provider_keyvault_secret +} + ## AKS output "cluster_name" { diff --git a/deployment/terraform/resources/providers.tf b/deployment/terraform/resources/providers.tf index 1f30b330..a369e401 100644 --- a/deployment/terraform/resources/providers.tf +++ b/deployment/terraform/resources/providers.tf @@ -10,7 +10,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "3.97.1" + version = "3.103.1" } } } diff --git a/deployment/terraform/resources/variables.tf b/deployment/terraform/resources/variables.tf index ba6f2aa3..61e2d58a 100644 --- a/deployment/terraform/resources/variables.tf +++ b/deployment/terraform/resources/variables.tf @@ -6,6 +6,19 @@ variable "region" { type = string } +# Ingress +variable "secret_provider_keyvault_name" { + type = string + description = "The name of the KeyVault that holds the secrets" + default = "pc-deploy-secrets" +} + +variable "secret_provider_keyvault_secret" { + type = string + description = "The name of the certificate in the KeyVault for TLS ingress" + default = "planetarycomputer-test-certificate" +} + # APIM variable "apim_sku_name" { diff --git a/docker-compose.console.yml b/docker-compose.console.yml index d0c9a344..eb057fc5 100644 --- a/docker-compose.console.yml +++ b/docker-compose.console.yml @@ -7,6 +7,7 @@ services: dockerfile: Dockerfile.dev volumes: - .:/opt/src + - ~/.azure:/root/.azure environment: - PCTASKS_CLIENT__ENDPOINT=http://nginx/tasks - PCTASKS_CLIENT__API_KEY=hunter2 @@ -111,3 +112,4 @@ services: networks: default: name: pctasks-network + external: true diff --git a/docker-compose.yml b/docker-compose.yml index 8ddc3ddb..738113bc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: "2.1" services: azurite: container_name: pctasks-azurite - image: mcr.microsoft.com/azure-storage/azurite:3.27.0 + image: mcr.microsoft.com/azure-storage/azurite:3.30.0 hostname: azurite command: "azurite --silent --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 -l /workspace"