Use Key Vault secrets instead of cert-manager secret in AKS cluster (#…

…185)

* Update deployment script and add nginx-values.yaml

* Update Dockerfile, values.yaml, variables.tf, and lib script

* Update to satisfy pydantic and mypy

* More mypy induced changes

* More mypy changes

deployment/Dockerfile

@@ -33,7 +33,7 @@ RUN apt-get install -y kubectl
 RUN curl https://baltocdn.com/helm/signing.asc |  apt-key add -
 RUN echo "deb https://baltocdn.com/helm/stable/debian/ all main" | tee /etc/apt/sources.list.d/helm-stable-debian.list
 RUN apt-get update
-RUN apt-get install helm=3.5.0-1
+RUN apt-get install helm=3.14.0-1
 
 # Install kubelogin
 

deployment/bin/deploy

@@ -138,14 +138,14 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
 
     # Install cert-manager
 
-    echo "Installing cert-manager..."
+    # echo "Installing cert-manager..."
 
-    helm upgrade --install \
-        cert-manager \
-        --namespace pc \
-        --create-namespace \
-        --version v1.6.0 \
-        --set installCRDs=true jetstack/cert-manager
+    # helm upgrade --install \
+    #     cert-manager \
+    #     --namespace pc \
+    #     --create-namespace \
+    #     --version v1.6.0 \
+    #     --set installCRDs=true jetstack/cert-manager
 
     echo "==================="
     echo "==== STAC API ====="
@@ -191,7 +191,8 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
         --set controller.service.loadBalancerIP="${INGRESS_IP}" \
         --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-dns-label-name"="${DNS_LABEL}" \
         --wait \
-        --timeout 2m0s
+        --timeout 2m0s \
+        -f bin/nginx-values.yaml
 
     #########################
     # Deploy Azure Function #

deployment/bin/lib

@@ -43,6 +43,10 @@ function gather_tf_output() {
     export ENVIRONMENT=$(tf_output environment)
     export INGRESS_IP=$(tf_output ingress_ip)
     export DNS_LABEL=$(tf_output dns_label)
+    export AZURE_TENANT=$(tf_output tenant_id)
+    export KEYVAULT_NAME=$(tf_output secret_provider_keyvault_name)
+    export SECRET_PROVIDER_MANAGED_IDENTITY_ID=$(tf_output secret_provider_managed_identity_id)
+    export SECRET_PROVIDER_KEYVAULT_SECRET=$(tf_output secret_provider_keyvault_secret)
 
     export FUNCTION_APP_NAME=$(tf_output function_app_name)
 
@@ -53,7 +57,7 @@ function gather_tf_output() {
 
 function render_values() {
     echo "Rendering chart value files..."
-
+    
     bin/jinja ${TF_OUTPUT_FILE} ${TEMPLATE_PATH} ${DEPLOY_VALUES_FILE}
 }
 

deployment/bin/nginx-values.yaml

@@ -0,0 +1,16 @@
+controller:
+  podLabels:
+    azure.workload.identity/use: "true"
+  extraVolumes:
+    - name: secrets-store-inline
+      csi:
+        driver: secrets-store.csi.k8s.io
+        readOnly: true
+        volumeAttributes:
+          secretProviderClass: "keyvault"
+  extraVolumeMounts:
+    - name: secrets-store-inline
+      mountPath: "/mnt/secrets-store"
+      readOnly: true
+  extraArgs:
+    default-ssl-certificate: pc/planetarycomputer-test-certificate

deployment/helm/deploy-values.template.yaml

@@ -73,8 +73,8 @@ tiler:
 
   stac_api_url: http://planetary-computer-stac.pc.svc.cluster.local
   stac_api_href: stac/
-  # PCT sas needs to be accessed through api management
-  pc_sdk_sas_url: https://pct-sas-westeurope-staging-apim.azure-api.net/sas/token
+  # PCT sas needs to be accessed through Azure Front Door
+  pc_sdk_sas_url: https://planetarycomputer-test.microsoft.com/sas/token
   pc_sdk_subscription_key: "{{ tf.pc_sdk_subscription_key }}"
   vectortile_sa_base_url: https://pcvectortiles.blob.core.windows.net
 
@@ -111,21 +111,19 @@ pcingress:
     secretName: "pqe-tls-secret"
 
   certIssuer:
-    enabled: true
+    enabled: false
     privateKeySecretRef: "{{ tf.cluster_cert_issuer }}"
     server: "{{ tf.cluster_cert_server }}"
     issuerEmail: "[email protected]"
     secretName: "pqe-tls-secret"
 
   ingress:
     enabled: true
-    tlsHost: "{{ tf.dns_label }}.{{ tf.location }}.cloudapp.azure.com"
+    tlsHost: "planetarycomputer-test.microsoft.com"
     hosts:
-      - "{{ tf.dns_label }}.{{ tf.location }}.cloudapp.azure.com"
       - "planetarycomputer-test.microsoft.com"
     annotations:
       kubernetes.io/ingress.class: nginx
-      cert-manager.io/cluster-issuer: "{{ tf.cluster_cert_issuer }}-pcingress"
       nginx.ingress.kubernetes.io/rewrite-target: /$2
       nginx.ingress.kubernetes.io/use-regex: "true"
       nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"
@@ -146,3 +144,12 @@ postgres:
   user: "{{ tf.pg_user }}"
   password: "{{ tf.pg_password }}"
   dbName: "{{ tf.pg_database }}"
+
+secretProvider:
+  create: true
+  providerName: "keyvault"
+  userAssignedIdentityID: "{{ env.SECRET_PROVIDER_MANAGED_IDENTITY_ID }}"
+  tenantId: "{{ env.AZURE_TENANT }}"
+  keyvaultName: "{{ env.KEYVAULT_NAME }}"
+  keyvaultCertificateName: "{{ env.SECRET_PROVIDER_KEYVAULT_SECRET }}"
+  kubernetesCertificateSecretName: "{{ env.SECRET_PROVIDER_KEYVAULT_SECRET }}"

deployment/helm/pc-apis-ingress/templates/cluster_issuer.yaml

@@ -1,20 +0,0 @@
-{{- if .Values.pcingress.certIssuer.enabled -}}
-apiVersion: cert-manager.io/v1
-kind: ClusterIssuer
-metadata:
-  name: {{ .Values.pcingress.certIssuer.privateKeySecretRef }}-pcingress
-spec:
-  acme:
-    server: {{ .Values.pcingress.certIssuer.server }}
-    email: {{ .Values.pcingress.certIssuer.issuerEmail }}
-    privateKeySecretRef:
-      name: {{ .Values.pcingress.certIssuer.privateKeySecretRef }}
-    solvers:
-    - http01:
-        ingress:
-          class: nginx
-          podTemplate:
-            spec:
-              nodeSelector:
-                "kubernetes.io/os": linux
-{{- end }}

deployment/helm/pc-apis-ingress/templates/ingress.yaml

@@ -18,7 +18,7 @@ spec:
   tls:
   - hosts:
     - {{ .Values.pcingress.ingress.tlsHost }}
-    secretName: {{ .Values.pcingress.cert.secretName }}
+    secretName: {{ .Values.secretProvider.kubernetesCertificateSecretName }}
   rules:
   {{- range .Values.pcingress.ingress.hosts }}
     - host: {{ . }}

deployment/helm/pc-apis-ingress/templates/secret-provider.yaml

@@ -0,0 +1,28 @@
+{{- if .Values.secretProvider.create -}}
+apiVersion: secrets-store.csi.x-k8s.io/v1
+kind: SecretProviderClass
+metadata:
+  name: {{ .Values.secretProvider.providerName }}
+  namespace: {{ .Values.namespace }}
+spec:
+  provider: azure
+  secretObjects:
+  - secretName: {{ .Values.secretProvider.kubernetesCertificateSecretName }}
+    type: kubernetes.io/tls
+    data:
+    - objectName: {{ .Values.secretProvider.keyvaultCertificateName }}
+      key: tls.crt
+    - objectName: {{ .Values.secretProvider.keyvaultCertificateName }}
+      key: tls.key
+  parameters:
+    usePodIdentity: "false"
+    clientID: "{{ .Values.secretProvider.userAssignedIdentityID }}"
+    keyvaultName: "{{ .Values.secretProvider.keyvaultName }}"
+    tenantId: "{{ .Values.secretProvider.tenantId }}"
+    cloudName: ""
+    objects:  |
+      array:
+        - |
+          objectName: {{ .Values.secretProvider.keyvaultCertificateName }}
+          objectType: secret
+{{- end }}

deployment/helm/pc-apis-ingress/values.yaml

@@ -7,6 +7,15 @@ stac:
 tiler:
   enabled: true
 
+secretProvider:
+  create: true
+  providerName: "keyvault"
+  userAssignedIdentityID: ""
+  tenantId: ""
+  keyvaultName: ""
+  keyvaultCertificateName: ""
+  kubernetesCertificateSecretName: ""
+
 pcingress:
   services:
     stac:

deployment/terraform/resources/aks.tf

@@ -5,6 +5,19 @@ resource "azurerm_kubernetes_cluster" "pc" {
   dns_prefix          = "${local.prefix}-cluster"
   kubernetes_version  = var.k8s_version
 
+  key_vault_secrets_provider {
+    secret_rotation_enabled = true
+  }
+  oidc_issuer_enabled = true
+
+   # https://learn.microsoft.com/en-us/azure/aks/auto-upgrade-cluster#use-cluster-auto-upgrade
+  automatic_channel_upgrade = "rapid"
+
+  # https://learn.microsoft.com/en-us/azure/aks/auto-upgrade-node-os-image
+  node_os_channel_upgrade = "NodeImage"
+
+  image_cleaner_enabled = true
+
   default_node_pool {
     name                 = "agentpool"
     os_sku               = "AzureLinux"
@@ -35,3 +48,22 @@ resource "azurerm_role_assignment" "network" {
   role_definition_name = "Network Contributor"
   principal_id         = azurerm_kubernetes_cluster.pc.identity[0].principal_id
 }
+
+resource "azurerm_federated_identity_credential" "cluster" {
+  name                = "federated-id-${local.prefix}-${var.environment}"
+  resource_group_name = azurerm_kubernetes_cluster.pc.node_resource_group
+  audience            = ["api://AzureADTokenExchange"]
+  issuer              = azurerm_kubernetes_cluster.pc.oidc_issuer_url
+  subject             = "system:serviceaccount:pc:nginx-ingress-ingress-nginx"
+  parent_id           = "/subscriptions/a84a690d-585b-4c7c-80d9-851a48af5a50/resourceGroups/MC_pct-apis-westeurope-staging_rg_pct-apis-westeurope-staging-cluster_westeurope/providers/Microsoft.ManagedIdentity/userAssignedIdentities/azurekeyvaultsecretsprovider-pct-apis-westeurope-staging-cluster"
+  timeouts {}
+}
+
+# If you add a second azurerm provider and use a data block to reference this key vault
+# then the identity that deploys this has to have permissions over both subscriptions
+# This role assignment was created manually but the resource is left here as a reminder
+# resource "azurerm_role_assignment" "certificateAccess" {
+#   scope                = #REDACTED
+#   role_definition_name = #REDACTED
+#   principal_id         = azurerm_kubernetes_cluster.pc.key_vault_secrets_provider[0].secret_identity[0].object_id
+# }

deployment/terraform/resources/output.tf

@@ -14,6 +14,10 @@ output "resource_group" {
   value = azurerm_resource_group.pc.name
 }
 
+output "tenant_id" {
+  value = data.azurerm_client_config.current.tenant_id
+}
+
 # -- Postgres
 
 output "pg_host" {
@@ -57,6 +61,18 @@ output "dns_label" {
   value = azurerm_public_ip.pc.domain_name_label
 }
 
+output "secret_provider_keyvault_name" {
+  value = var.secret_provider_keyvault_name
+}
+
+output "secret_provider_managed_identity_id" {
+  value = azurerm_kubernetes_cluster.pc.key_vault_secrets_provider[0].secret_identity[0].client_id
+}
+
+output "secret_provider_keyvault_secret" {
+  value = var.secret_provider_keyvault_secret
+}
+
 ## STAC API
 
 output "stac_replica_count" {

deployment/terraform/resources/providers.tf

@@ -12,3 +12,5 @@ terraform {
     }
   }
 }
+
+data "azurerm_client_config" "current" {}

deployment/terraform/resources/variables.tf

@@ -84,6 +84,18 @@ variable "pc_sdk_subscription_key_secret_name" {
   default = "pct-tiler-sdk-subscription-key"
 }
 
+variable "secret_provider_keyvault_name" {
+  type = string
+  description = "The name of the KeyVault that holds the secrets"
+  default = "pc-deploy-secrets"
+}
+
+variable "secret_provider_keyvault_secret" {
+  type = string
+  description = "The name of the certificate in the KeyVault for TLS ingress"
+  default = "planetarycomputer-test-certificate"
+}
+
 # -- Functions --
 
 variable "output_storage_account_name" {

pccommon/pccommon/config/__init__.py

@@ -24,4 +24,5 @@ def get_all_render_configs() -> Dict[str, DefaultRenderConfig]:
         for id, coll in get_apis_config()
         .get_collection_config_table()
         .get_all_configs()
+        if id is not None
     }

pccommon/pccommon/logging.py

@@ -61,7 +61,7 @@ def filter(self, record: logging.LogRecord) -> bool:
 # Prevent successful health check pings from being logged
 class HealthCheckFilter(logging.Filter):
     def filter(self, record: logging.LogRecord) -> bool:
-        if len(record.args) != 5:
+        if record.args is not None and len(record.args) != 5:
             return True
 
         args = cast(Tuple[str, str, str, str, int], record.args)

pccommon/pccommon/redis.py

@@ -114,7 +114,7 @@ async def cached_result(
     try:
         r = request.app.state.redis
         if r:
-            cached: str = await r.get(host_cache_key)
+            cached = await r.get(host_cache_key)
             if cached:
                 logger.info(
                     "Cache result hit",

pccommon/pccommon/tables.py

@@ -222,7 +222,7 @@ def get_all(self) -> Iterable[Tuple[Optional[str], Optional[str], M]]:
                 yield (
                     partition_key,
                     row_key,
-                    self._parse_model(entity, partition_key, row_key),
+                    self._parse_model(entity, partition_key, row_key),  # type: ignore
                 )
 
 

pccommon/setup.py

@@ -21,14 +21,8 @@
 ]
 
 extra_reqs = {
-    "test": [
-        "pytest",
-        "pytest-asyncio",
-    ],
-    "dev": [
-        "pytest",
-        "pytest-asyncio",
-    ],
+    "test": ["pytest", "pytest-asyncio", "types-redis"],
+    "dev": ["pytest", "pytest-asyncio", "types-redis"],
 }
 
 setup(

pcstac/Dockerfile.dev

@@ -3,4 +3,4 @@ FROM pc-apis-stac
 COPY requirements-dev.txt requirements-dev.txt
 RUN pip install -r requirements-dev.txt
 
-RUN pip install -e ./pccommon -e ./pcstac
+RUN pip install -e ./pccommon[dev] -e ./pcstac

pcstac/pcstac/config.py

@@ -64,11 +64,11 @@ class BackPressureConfig(BaseModel):
 
 
 class BackPressures(BaseSettings):
-    collections: BackPressureConfig
-    collection: BackPressureConfig
-    item: BackPressureConfig
-    items: BackPressureConfig
-    search: BackPressureConfig
+    collections: BackPressureConfig = BackPressureConfig()
+    collection: BackPressureConfig = BackPressureConfig()
+    item: BackPressureConfig = BackPressureConfig()
+    items: BackPressureConfig = BackPressureConfig()
+    search: BackPressureConfig = BackPressureConfig()
 
 
 class Settings(BaseSettings):
@@ -96,8 +96,8 @@ class Settings(BaseSettings):
     db_min_conn_size: int = Field(env=DB_MIN_CONN_ENV_VAR, default=1)
     openapi_url: str = "/openapi.json"
     api_version: str = f"v{API_VERSION}"
-    rate_limits: RateLimits
-    back_pressures: BackPressures
+    rate_limits: RateLimits = RateLimits()
+    back_pressures: BackPressures = BackPressures()
     request_timout: int = Field(env=REQUEST_TIMEOUT_ENV_VAR, default=30)
 
     def get_tiler_href(self, request: Request) -> str: