chore: simplify fl scripts

platforms/gke/base/use-cases/federated-learning/README.md

@@ -1,9 +1,6 @@
 # Federated learning on Google Cloud
 
-## Deploy the Federated Learning reference architecture
-
-1. Provision the base platform by following the
-   [Core GKE Accelerated Platform guide](/platforms/gke/base/core/README.md).
+## Deploy the Federated learning reference architecture
 
 1. Provision the Federated Learning reference architecture:
 
@@ -18,6 +15,3 @@
    ```sh
    "${ACP_PLATFORM_BASE_DIR}/use-cases/federated-learning/teardown.sh"
    ```
-
-1. Teardown the base platform by following the
-   [Core GKE Accelerated Platform guide](/platforms/gke/base/core/README.md#teardown).

platforms/gke/base/use-cases/federated-learning/common.sh

@@ -18,13 +18,20 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+ACP_PLATFORM_SHARED_CONFIG_DIR="${ACP_PLATFORM_BASE_DIR}/_shared_config"
+
+# shellcheck disable=SC2034 # Variable is used in other scripts
+ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE="${ACP_PLATFORM_SHARED_CONFIG_DIR}/cluster.auto.tfvars"
+
 # shellcheck disable=SC1091
-source "${ACP_PLATFORM_BASE_DIR}/_shared_config/scripts/set_environment_variables.sh" "${ACP_PLATFORM_BASE_DIR}/_shared_config"
+source "${ACP_PLATFORM_SHARED_CONFIG_DIR}/scripts/set_environment_variables.sh" "${ACP_PLATFORM_SHARED_CONFIG_DIR}"
 
 FEDERATED_LEARNING_USE_CASE_DIR="${ACP_PLATFORM_BASE_DIR}/use-cases/federated-learning"
 
 # shellcheck disable=SC2034 # Variable is used in other scripts
 FEDERATED_LEARNING_USE_CASE_TERRAFORM_DIR="${FEDERATED_LEARNING_USE_CASE_DIR}/terraform"
+# shellcheck disable=SC2034 # Variable is used in other scripts
+FEDERATED_LEARNING_SHARED_CONFIG_DIR="${FEDERATED_LEARNING_USE_CASE_TERRAFORM_DIR}/_shared_config"
 
 # shellcheck disable=SC2034 # Variable is used in other scripts
 FEDERATED_LEARNING_USE_CASE_INITIALIZE_SERVICE_DIR="${FEDERATED_LEARNING_USE_CASE_TERRAFORM_DIR}/initialize"
@@ -35,3 +42,60 @@ federated_learning_terraservices=(
   "container_image_repository"
   "private_google_access"
 )
+
+TERRAFORM_INIT_COMMAND=(
+  terraform init
+)
+
+# shellcheck disable=SC2034 # Variable is used in other scripts
+TERRAFORM_INIT_BACKEND_CONFIG_COMMAND=(
+  "${TERRAFORM_INIT_COMMAND[@]}"
+  -backend-config="backend.config"
+)
+
+# shellcheck disable=SC2034 # Variable is used in other scripts
+TERRAFORM_CLUSTER_CONFIGURATION=(
+)
+
+provision_terraservice() {
+  local terraservice
+  terraservice="${1}"
+
+  local -a TERRASERVICE_TERRAFORM_INIT_COMMAND
+
+  echo "Provisioning ${terraservice}"
+  TERRASERVICE_TERRAFORM_INIT_COMMAND=(
+    "${TERRAFORM_INIT_BACKEND_CONFIG_COMMAND[@]}"
+  )
+
+  if [ "${terraservice:-}" == "initialize" ]; then
+    TERRASERVICE_TERRAFORM_INIT_COMMAND=(
+      "${TERRAFORM_INIT_COMMAND[@]}"
+    )
+  fi
+
+  echo "Terraform init command for ${terraservice}: ${TERRASERVICE_TERRAFORM_INIT_COMMAND[*]}"
+
+  cd "${FEDERATED_LEARNING_USE_CASE_TERRAFORM_DIR}/${terraservice}" &&
+    echo "Current directory: $(pwd)" &&
+    "${TERRASERVICE_TERRAFORM_INIT_COMMAND[@]}" &&
+    terraform plan -input=false -out=tfplan &&
+    terraform apply -input=false tfplan
+  local _apply_result=$?
+  rm tfplan
+  if [[ ${_apply_result} -ne 0 ]]; then
+    echo "Terraform apply for ${terraservice} failed with code ${_apply_result}"
+    exit $_apply_result
+  fi
+}
+
+get_terraform_output() {
+  terraservice="${1}"
+  output_name="${2}"
+  local output
+  if ! output="$(cd "${FEDERATED_LEARNING_USE_CASE_TERRAFORM_DIR}/${terraservice}" && terraform output -raw "${output_name}")"; then
+    echo "Error while getting ${output_name} output"
+    return 1
+  fi
+  echo "${output}"
+}

platforms/gke/base/use-cases/federated-learning/deploy.sh

@@ -23,36 +23,21 @@ source "${ACP_PLATFORM_BASE_DIR}/use-cases/federated-learning/common.sh"
 
 start_timestamp_federated_learning=$(date +%s)
 
-TERRAFORM_INIT_COMMAND=(
-  terraform init
-)
+echo "Preparing core platform configuration files"
+for configuration_variable in "${TERRAFORM_CLUSTER_CONFIGURATION[@]}"; do
+  configuration_variable_name="$(echo "${configuration_variable}" | awk '{ print $1 }')"
+  echo "Checking if ${configuration_variable_name} is in ${ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE}"
+  grep -q "${configuration_variable_name}" "${ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE}" || echo "${configuration_variable}" >>"${ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE}"
+done
+terraform fmt "${ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE}"
+
+echo "Provisioning the core platform"
+"${ACP_PLATFORM_CORE_DIR}/deploy.sh"
 
+echo "Provisioning the use case resources"
 # shellcheck disable=SC2154 # variable defined in common.sh
 for terraservice in "${federated_learning_terraservices[@]}"; do
-  echo "Provisioning ${terraservice}"
-  TERRASERVICE_TERRAFORM_INIT_COMMAND=(
-    "${TERRAFORM_INIT_COMMAND[@]}"
-  )
-  if [ "${terraservice:-}" != "initialize" ]; then
-    echo "Add the option to load remote backend configuration to the terraform init command"
-    TERRASERVICE_TERRAFORM_INIT_COMMAND+=(
-      -backend-config="backend.config"
-    )
-  fi
-
-  echo "Terraform init command: ${TERRASERVICE_TERRAFORM_INIT_COMMAND[*]}"
-
-  cd "${FEDERATED_LEARNING_USE_CASE_TERRAFORM_DIR}/${terraservice}" &&
-    echo "Current directory: $(pwd)" &&
-    "${TERRASERVICE_TERRAFORM_INIT_COMMAND[@]}" &&
-    terraform plan -input=false -out=tfplan &&
-    terraform apply -input=false tfplan
-  _apply_result=$?
-  rm tfplan
-  if [[ ${_apply_result} -ne 0 ]]; then
-    echo "Terraform apply failed with code ${_apply_result}"
-    exit $_apply_result
-  fi
+  provision_terraservice "${terraservice}"
 done
 
 end_timestamp_federated_learning=$(date +%s)

platforms/gke/base/use-cases/federated-learning/teardown.sh

@@ -18,21 +18,34 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-start_timestamp_federated_learning=$(date +%s)
-
 # shellcheck disable=SC1091
 source "${ACP_PLATFORM_BASE_DIR}/use-cases/federated-learning/common.sh"
 
+start_timestamp_federated_learning=$(date +%s)
+
+# Iterate over the terraservices array so we destroy them in reverse order, keeping the
+# initialize terraservice last.
 # shellcheck disable=SC2154 # variable defined in common.sh
-for terraservice in "${federated_learning_terraservices[@]}"; do
+for ((i = ${#federated_learning_terraservices[@]} - 1; i >= 0; i--)); do
+  terraservice=${federated_learning_terraservices[i]}
+  echo "Destroying ${terraservice}"
+
+  TERRASERVICE_TERRAFORM_INIT_COMMAND=(
+    "${TERRAFORM_INIT_BACKEND_CONFIG_COMMAND[@]}"
+  )
+
+  # The initialize terraservice uses a local backend, so we don't add the remote backend configuration
   if [ "${terraservice:-}" == "initialize" ]; then
-    echo "Skip destroying ${terraservice} to avoid removing backend configuration files"
-    continue
+    TERRASERVICE_TERRAFORM_INIT_COMMAND=(
+      "${TERRAFORM_INIT_COMMAND[@]}"
+    )
   fi
-  echo "Destroying ${terraservice}"
+
+  echo "Terraform init command: ${TERRASERVICE_TERRAFORM_INIT_COMMAND[*]}"
+
   cd "${FEDERATED_LEARNING_USE_CASE_TERRAFORM_DIR}/${terraservice}" &&
     echo "Current directory: $(pwd)" &&
-    terraform init -backend-config="backend.config" &&
+    "${TERRASERVICE_TERRAFORM_INIT_COMMAND[@]}" &&
     terraform destroy -auto-approve
 
   _destroy_result=$?
@@ -45,6 +58,15 @@ for terraservice in "${federated_learning_terraservices[@]}"; do
   fi
 done
 
+echo "Destroying the core platform"
+"${ACP_PLATFORM_CORE_DIR}/teardown.sh"
+
+for configuration_variable in "${TERRAFORM_CLUSTER_CONFIGURATION[@]}"; do
+  configuration_variable_name="$(echo "${configuration_variable}" | awk ' { print $1 }'))"
+  sed -i "/${configuration_variable_name}/d" "${ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE}"
+done
+terraform fmt "${ACP_PLATFORM_SHARED_CONFIG_CLUSTER_AUTO_VARS_FILE}"
+
 end_timestamp_federated_learning=$(date +%s)
 total_runtime_value_federated_learning=$((end_timestamp_federated_learning - start_timestamp_federated_learning))
 echo "Total runtime (Federated learning use case provisioning and configuration): $(date -d@${total_runtime_value_federated_learning} -u +%H:%M:%S)"

platforms/gke/base/use-cases/federated-learning/terraform/initialize/main.tf

@@ -19,16 +19,16 @@ locals {
     bucket = data.google_storage_bucket.terraform.name
   })
 
-  use_case_terraservices = [
-    # Don't configure the remote backend for the initialize terraservice
-    # because we use a local backend for that in order to solve a
-    # chicken-and-egg issue where the initialize terraservice depends on
-    # the configuration files that the initialize terraservice creates.
-    # Keeping this as a reference if we change idea in the future
-    # abspath("${path.module}"),
-    abspath("${path.module}/../container_image_repository"),
-    abspath("${path.module}/../private_google_access"),
-  ]
+  terraservices_path = "${path.module}/.."
+
+  # Search for backend.tf files, assuming that a directory that contains a
+  # backend.tf file is a terraservice to provision.
+  # This search excludes the initialize terraservice
+  # because we use a local backend for that in order to solve a
+  # chicken-and-egg issue where the initialize terraservice depends on
+  # the configuration files that the initialize terraservice creates.
+  backend_files          = flatten([for _, v in flatten(fileset("${local.terraservices_path}/", "**/backend.tf")) : v])
+  use_case_terraservices = toset([for _, v in local.backend_files : abspath("${local.terraservices_path}/${trimprefix(trimsuffix(dirname(v), "/backend.tf"), "../")}")])
 }
 
 data "google_project" "default" {

test/ci-cd/cloudbuild/uc-federated-learning-terraform-destroy.yaml

@@ -51,8 +51,16 @@ steps:
         export TF_VAR_platform_name="${_PLATFORM_NAME}"
         export TF_VAR_terraform_project_id="${PROJECT_ID}"
 
-        platforms/gke/base/use-cases/federated-learning/teardown.sh
-        platforms/gke/base/core/teardown.sh
+        # Deploy the base platform and the use case
+        platforms/gke/base/use-cases/federated-learning/deploy.sh
+
+        APPLY_RETURN_CODE=$?
+        echo "APPLY_RETURN_CODE=$${APPLY_RETURN_CODE}"
+
+        platforms/gke/base/use-cases/federated-learning/teardown.sh || exit 1
+        if [[ $${APPLY_RETURN_CODE} -ne 0 ]]; then
+          exit $${APPLY_RETURN_CODE}
+        fi
     waitFor:
       - "Build runner image"
 

test/ci-cd/cloudbuild/uc-federated-learning-terraform.yaml

@@ -52,14 +52,12 @@ steps:
         export TF_VAR_terraform_project_id="${PROJECT_ID}"
 
         # Deploy the base platform and the use case
-        platforms/gke/base/core/deploy.sh &&
         platforms/gke/base/use-cases/federated-learning/deploy.sh
 
         APPLY_RETURN_CODE=$?
         echo "APPLY_RETURN_CODE=$${APPLY_RETURN_CODE}"
 
         platforms/gke/base/use-cases/federated-learning/teardown.sh || exit 1
-        platforms/gke/base/core/teardown.sh || exit 1
         if [[ $${APPLY_RETURN_CODE} -ne 0 ]]; then
           exit $${APPLY_RETURN_CODE}
         fi