Merge branch 'main' into user/elay/add-function-app-to-generate-banne…

…d-ip

.github/workflows/cicd.yml

@@ -32,7 +32,7 @@ jobs:
 
       - name: Get image tag
         id: get_image_tag
-        run: 
+        run:
           case "${GITHUB_REF}" in
             *tags*)
               echo "tag=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT ;
@@ -57,7 +57,7 @@ jobs:
       - build_and_publish
     steps:
       - uses: actions/checkout@v3
-      
+
       - name: Log in with Azure
         uses: azure/login@v1
         with:
@@ -86,4 +86,4 @@ jobs:
           ARM_CLIENT_ID: ${{ fromJSON(secrets.SECURE_AZURE_CREDENTIALS).clientId }}
           ARM_SUBSCRIPTION_ID: ${{ fromJSON(secrets.SECURE_AZURE_CREDENTIALS).subscriptionId }}
           ARM_TENANT_ID: ${{ fromJSON(secrets.SECURE_AZURE_CREDENTIALS).tenantId }}
-          ARM_USE_OIDC: true
+          ARM_USE_OIDC: true

deployment/README.md

@@ -10,6 +10,8 @@ The logic for the deployment workflow is encapsulated in the [bin/deploy](bin/de
 scripts/console --deploy
 ```
 
+To have access to the remote backend terraform state, the identity (App Registration in CI, or local corp credential if local) will need to have the `Storage Blob Data Owner` role on the `pctesttfstate` storage account.
+
 ## Manual resources
 
 ### Deployment secrets Key Vault
@@ -40,25 +42,27 @@ Container Registry repo where you published your local images:
 - `ACR_TILER_REPO`
 - `IMAGE_TAG`
 
-__Note:__ Remember to bring down your resources after testing with `terraform destroy`!
+**Note:** Remember to bring down your resources after testing with `terraform destroy`!
 
 ## Loading configuration data
 
 Configuration data is stored in Azure Storage Tables. Use the `pcapis` command line interface that is installed with the `pccommon` package to load data. For example:
 
+```console
+>  az login # Use an account that has "Storage Table Data Contributor" on the account
+>  pcapis load -t collection --account pctapissatyasa  --table collectionconfig --file pccommon/tests/data-files/collection_config.json
 ```
->  pcapis load -t collection --sas "${SAS_TOKEN}" --account pctapissatyasa  --table collectionconfig --file pccommon/tests/data-files/collection_config.json
-```
+
 To dump a single collection config, use:
 
-```
->  pcapis dump -t collection --sas "${SAS_TOKEN}" --account pctapissatyasa  --table collectionconfig --id naip
+```console
+>  pcapis dump -t collection  --account pctapissatyasa  --table collectionconfig --id naip
 ```
 
 For container configs, you must also specify the container account name used as the Partition Key:
 
-```
->  pcapis dump -t collection --sas "${SAS_TOKEN}" --account pctapissatyasa  --table containerconfig --id naip --container-account naipeuwest
+```console
+>  pcapis dump -t collection --account pctapissatyasa  --table containerconfig --id naip --container-account naipeuwest
 ```
 
 Using the `load` command on a single dump file for either config will update the single row.

deployment/bin/deploy

@@ -49,10 +49,21 @@ while [[ "$#" -gt 0 ]]; do case $1 in
         ;;
     esac done
 
+# Always disable shared access keys on script exit
+trap disable_shared_access_keys EXIT
+
 ###################################
 # Check and configure environment #
 ###################################
 
+# Enable shared access keys on storage accounts that must have properties read
+# [storage_account]=resource_group
+declare -A SAK_STORAGE_ACCOUNTS
+SAK_STORAGE_ACCOUNTS=(
+  ["pctapisstagingsa"]="pct-apis-westeurope-staging_rg"
+  ["pcfilestest"]="pc-test-manual-resources"
+)
+
 if [[ -z ${TERRAFORM_DIR} ]]; then
     echo "Must pass in TERRAFORM_DIR with -t"
     exit 1
@@ -73,6 +84,12 @@ setup_env
 echo "===== Running Deploy ====="
 echo "IMAGE_TAG: ${IMAGE_TAG}"
 
+if [ -z "$ARM_CLIENT_ID" ]; then
+  export ARM_CLIENT_ID=$(az account show --query user.name -o tsv)
+  echo "Using Azure CLI auth with username: ${ARM_CLIENT_ID}"
+fi
+
+
 # ---------------------------------------------------
 
 if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
@@ -91,6 +108,9 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
 
     if [[ "${SKIP_TF}" != 1 ]]; then
         echo "Deploying infrastructure with Terraform..."
+
+        enable_shared_access_keys
+
         terraform init --upgrade
 
         if [ "${PLAN_ONLY}" ]; then
@@ -142,7 +162,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
         --kube-context "${KUBE_CONTEXT}" \
         --wait \
         --timeout 2m0s \
-        -f ${DEPLOY_VALUES_FILE}
+        -f ${DEPLOY_VALUES_FILE} \
 
     echo "================"
     echo "==== Tiler ====="
@@ -154,7 +174,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
         --kube-context "${KUBE_CONTEXT}" \
         --wait \
         --timeout 2m0s \
-        -f ${DEPLOY_VALUES_FILE}
+        -f ${DEPLOY_VALUES_FILE} \
 
     echo "=================="
     echo "==== Ingress ====="

deployment/bin/kv_add_ip

@@ -32,6 +32,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
         -g ${KEY_VAULT_RESOURCE_GROUP_NAME}  \
         -n ${KEY_VAULT_NAME} \
         --ip-address $cidr \
-        --subscription ${ARM_SUBSCRIPTION_ID}
+        --subscription ${ARM_SUBSCRIPTION_ID} \
+        --output none
 
 fi

deployment/bin/kv_rmv_ip

@@ -32,6 +32,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
         -g ${KEY_VAULT_RESOURCE_GROUP_NAME}  \
         -n ${KEY_VAULT_NAME} \
         --ip-address $cidr \
-        --subscription ${ARM_SUBSCRIPTION_ID}
+        --subscription ${ARM_SUBSCRIPTION_ID} \
+        --output none
 
 fi

deployment/bin/lib

@@ -131,3 +131,43 @@ function get_cidr_range() {
     IFS='.' read -r -a ip_parts <<< "$runnerIpAddress"
     echo "${ip_parts[0]}.${ip_parts[1]}.0.0/16"
 }
+
+function disable_shared_access_keys() {
+    echo "Disabling shared access key on storage account..."
+
+    for SAK_STORAGE_ACCOUNT in "${!SAK_STORAGE_ACCOUNTS[@]}"; do
+        SAK_RESOURCE_GROUP=${SAK_STORAGE_ACCOUNTS[$SAK_STORAGE_ACCOUNT]}
+
+        az storage account update \
+            --name ${SAK_STORAGE_ACCOUNT} \
+            --resource-group ${SAK_RESOURCE_GROUP} \
+            --allow-shared-key-access false \
+            --output none
+
+        if [ $? -ne 0 ]; then
+            echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+            echo "WARNING: Failed to turn off shared key access on the storage account."
+            echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+            exit 2
+        fi
+    done
+}
+
+function enable_shared_access_keys() {
+    echo "Enabling shared key access for storage account..."
+    # Terraform isn't able to read all resources from a storage account if shared key access is disabled
+    # so while we're deploying, we need to enable it. Since we haven't run TF yet, we don't have the name of the account
+    # so they are hardcoded here. This is a temporary workaround until this is resolved
+    # https://github.com/hashicorp/terraform-provider-azurerm/issues/25218
+
+    for SAK_STORAGE_ACCOUNT in "${!SAK_STORAGE_ACCOUNTS[@]}"; do
+        SAK_RESOURCE_GROUP=${SAK_STORAGE_ACCOUNTS[$SAK_STORAGE_ACCOUNT]}
+
+        echo "   - enabling ${SAK_STORAGE_ACCOUNT} / ${SAK_RESOURCE_GROUP}"
+        az storage account update \
+            --name ${SAK_STORAGE_ACCOUNT} \
+            --resource-group ${SAK_RESOURCE_GROUP} \
+            --allow-shared-key-access true \
+            --output none
+    done
+}

deployment/docker-compose.yml

@@ -8,11 +8,11 @@ services:
     environment:
       - ACR_STAC_REPO=${ACR_STAC_REPO:-pccomponentstest.azurecr.io/planetary-computer-apis/stac}
       - ACR_TILER_REPO=${ACR_TILER_REPO:-pccomponentstest.azurecr.io/planetary-computer-apis/tiler}
-      - IMAGE_TAG
+      - IMAGE_TAG=${IMAGE_TAG:-latest}
       - GIT_COMMIT
 
       - ARM_SUBSCRIPTION_ID=${ARM_SUBSCRIPTION_ID:-a84a690d-585b-4c7c-80d9-851a48af5a50}
-      - ARM_TENANT_ID
+      - ARM_TENANT_ID=${ARM_TENANT_ID:-72f988bf-86f1-41af-91ab-2d7cd011db47}
       - ARM_CLIENT_ID
       - ARM_USE_OIDC
       - ARM_OIDC_TOKEN

deployment/helm/deploy-values.template.yaml

@@ -42,6 +42,11 @@ stac:
     replicaCount: "{{ tf.stac_replica_count }}"
     podAnnotations:
       "pc/gitsha": "{{ env.GIT_COMMIT }}"
+    useWorkloadIdentity: true
+    serviceAccount:
+      annotations:
+        "azure.workload.identity/client-id": {{ tf.cluster_stac_identity_client_id }}
+        "azure.workload.identity/tenant-id": {{ tf.tenant_id }}
 
   appRootPath: "/stac"
   port: "80"
@@ -86,7 +91,6 @@ tiler:
 
 storage:
   account_name: "{{ tf.storage_account_name }}"
-  account_key: "{{ tf.storage_account_key }}"
   collection_config_table_name: "{{ tf.collection_config_table_name }}"
   container_config_table_name: "{{ tf.container_config_table_name }}"
   ip_exception_config_table_name: "{{ tf.ip_exception_config_table_name }}"

deployment/helm/published/planetary-computer-stac/templates/_helpers.tpl

@@ -42,6 +42,7 @@ app.kubernetes.io/instance: {{ .Release.Name }}
 Common labels
 */}}
 {{- define "pcstac.labels" -}}
+azure.workload.identity/use: {{ .Values.stac.deploy.useWorkloadIdentity | quote}}
 helm.sh/chart: {{ include "pcstac.chart" . }}
 {{ include "pcstac.selectorLabels" . }}
 {{- if .Chart.AppVersion }}

deployment/helm/published/planetary-computer-stac/templates/deployment.yaml

@@ -19,7 +19,7 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       labels:
-        {{- include "pcstac.selectorLabels" . | nindent 8 }}
+        {{- include "pcstac.labels" . | nindent 8 }}
     spec:
       {{- with .Values.stac.deploy.imagePullSecrets }}
       imagePullSecrets:
@@ -89,20 +89,14 @@ spec:
               value: "{{ .Values.stac.debug }}"
             - name: "PCAPIS_COLLECTION_CONFIG__ACCOUNT_NAME"
               value: "{{ .Values.storage.account_name }}"
-            - name: "PCAPIS_COLLECTION_CONFIG__ACCOUNT_KEY"
-              value: "{{ .Values.storage.account_key }}"
             - name: "PCAPIS_COLLECTION_CONFIG__TABLE_NAME"
               value: "{{ .Values.storage.collection_config_table_name }}"
             - name: "PCAPIS_CONTAINER_CONFIG__ACCOUNT_NAME"
               value: "{{ .Values.storage.account_name }}"
-            - name: "PCAPIS_CONTAINER_CONFIG__ACCOUNT_KEY"
-              value: "{{ .Values.storage.account_key }}"
             - name: "PCAPIS_CONTAINER_CONFIG__TABLE_NAME"
               value: "{{ .Values.storage.container_config_table_name }}"
             - name: "PCAPIS_IP_EXCEPTION_CONFIG__ACCOUNT_NAME"
               value: "{{ .Values.storage.account_name }}"
-            - name: "PCAPIS_IP_EXCEPTION_CONFIG__ACCOUNT_KEY"
-              value: "{{ .Values.storage.account_key }}"
             - name: "PCAPIS_IP_EXCEPTION_CONFIG__TABLE_NAME"
               value: "{{ .Values.storage.ip_exception_config_table_name }}"
             - name: "PCAPIS_REDIS_HOSTNAME"

deployment/helm/published/planetary-computer-stac/templates/serviceaccount.yaml

@@ -6,7 +6,7 @@ metadata:
   name: {{ include "pcstac.serviceAccountName" . }}
   labels:
     {{- include "pcstac.labels" . | nindent 4 }}
-  {{- with .Values.serviceAccount.annotations }}
+  {{- with .Values.stac.deploy.serviceAccount.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}
   {{- end }}

deployment/helm/published/planetary-computer-stac/values.yaml

@@ -56,6 +56,10 @@ stac:
     affinity: {}
     autoscaling:
       enabled: false
+    useWorkloadIdentity: false
+    serviceAccount:
+      annotations: {}
+
 
   cert:
     privateKeySecretRef: "letsencrypt-staging"

deployment/helm/published/planetary-computer-tiler/templates/deployment.yaml

@@ -85,20 +85,14 @@ spec:
               value: "{{ .Values.tiler.default_max_items_per_tile}}"
             - name: "PCAPIS_COLLECTION_CONFIG__ACCOUNT_NAME"
               value: "{{ .Values.storage.account_name }}"
-            - name: "PCAPIS_COLLECTION_CONFIG__ACCOUNT_KEY"
-              value: "{{ .Values.storage.account_key }}"
             - name: "PCAPIS_COLLECTION_CONFIG__TABLE_NAME"
               value: "{{ .Values.storage.collection_config_table_name }}"
             - name: "PCAPIS_CONTAINER_CONFIG__ACCOUNT_NAME"
               value: "{{ .Values.storage.account_name }}"
-            - name: "PCAPIS_CONTAINER_CONFIG__ACCOUNT_KEY"
-              value: "{{ .Values.storage.account_key }}"
             - name: "PCAPIS_CONTAINER_CONFIG__TABLE_NAME"
               value: "{{ .Values.storage.container_config_table_name }}"
             - name: "PCAPIS_IP_EXCEPTION_CONFIG__ACCOUNT_NAME"
               value: "{{ .Values.storage.account_name }}"
-            - name: "PCAPIS_IP_EXCEPTION_CONFIG__ACCOUNT_KEY"
-              value: "{{ .Values.storage.account_key }}"
             - name: "PCAPIS_IP_EXCEPTION_CONFIG__TABLE_NAME"
               value: "{{ .Values.storage.ip_exception_config_table_name }}"
             - name: "PCAPIS_TABLE_VALUE_TTL"

deployment/terraform/resources/aks.tf

@@ -26,6 +26,10 @@ resource "azurerm_kubernetes_cluster" "pc" {
     vm_size        = "Standard_DS2_v2"
     node_count     = var.aks_node_count
     vnet_subnet_id = azurerm_subnet.node_subnet.id
+
+    upgrade_settings {
+      max_surge = "10%"
+    }
   }
 
   identity {
@@ -74,9 +78,31 @@ resource "azurerm_kubernetes_cluster" "pc" {
   }
 }
 
+resource "azurerm_user_assigned_identity" "stac" {
+  name                = "id-${local.prefix}-stac"
+  location            = var.region
+  resource_group_name = azurerm_resource_group.pc.name
+}
+
+resource "azurerm_federated_identity_credential" "stac" {
+  name                = "federated-id-${local.prefix}"
+  resource_group_name = azurerm_resource_group.pc.name
+  audience            = ["api://AzureADTokenExchange"]
+  issuer              = azurerm_kubernetes_cluster.pc.oidc_issuer_url
+  subject             = "system:serviceaccount:pc:planetary-computer-stac"
+  parent_id           = azurerm_user_assigned_identity.stac.id
+  timeouts {}
+}
+
+resource "azurerm_role_assignment" "cluster-stac-identity-storage-access" {
+  scope                = azurerm_storage_account.pc.id
+  role_definition_name = "Storage Table Data Reader"
+  principal_id         = azurerm_user_assigned_identity.stac.principal_id
+}
+
 # Workload Identity for tiler access to the Azure Maps account
 resource "azurerm_user_assigned_identity" "tiler" {
-  name                = "id-${local.prefix}"
+  name                = "id-${local.prefix}-tiler"
   location            = var.region
   resource_group_name = azurerm_resource_group.pc.name
 }
@@ -98,6 +124,12 @@ resource "azurerm_role_assignment" "cluster-identity-maps-render-token" {
 
 }
 
+resource "azurerm_role_assignment" "cluster-tiler-identity-storage-access" {
+  scope                = azurerm_storage_account.pc.id
+  role_definition_name = "Storage Table Data Reader"
+  principal_id         = azurerm_user_assigned_identity.tiler.principal_id
+}
+
 # add the role to the identity the kubernetes cluster was assigned
 resource "azurerm_role_assignment" "network" {
   scope                = azurerm_resource_group.pc.id

deployment/terraform/resources/functions.tf

@@ -46,7 +46,7 @@ resource "azurerm_function_app" "pcfuncs" {
   os_type = "linux"
   version = "~4"
   site_config {
-    linux_fx_version          = "PYTHON|3.8"
+    linux_fx_version          = "PYTHON|3.9"
     use_32_bit_worker_process = false
     ftps_state                = "Disabled"
 

deployment/terraform/resources/output.tf

@@ -55,6 +55,10 @@ output "cluster_tiler_identity_client_id" {
   value = azurerm_user_assigned_identity.tiler.client_id
 }
 
+output "cluster_stac_identity_client_id" {
+  value = azurerm_user_assigned_identity.stac.client_id
+}
+
 ## Ingress
 
 output "ingress_ip" {
@@ -104,10 +108,6 @@ output "storage_account_name" {
   value = azurerm_storage_account.pc.name
 }
 
-output "storage_account_key" {
-  value = azurerm_storage_account.pc.primary_access_key
-}
-
 output "collection_config_table_name" {
   value = azurerm_storage_table.collectionconfig.name
 }